import matplotlib.pyplot as plt
import os
from utils import get_accuracy, get_samples, train_out_proj_fast, train_out_proj_closed_form
from main import Args
from data import load_SHD
from model import EchoSpike, simple_out
import numpy as np
from data import augment_shd
import torch
import seaborn as sns
from scipy.signal import savgol_filter
from tqdm.notebook import trange
from matplotlib import pyplot
pyplot.rcParams['figure.dpi'] = 600
plt.rcParams.update({'font.size': 18})
import pickle
torch.manual_seed(0)
color_list = sns.color_palette('muted')
device = 'cpu'
batch_size = 128
folder = 'models/'
model_name = 'shd_1layer_large.pt'
with open(folder + model_name[:-3] + '_args.pkl', 'rb') as f:
args = pickle.load(f)
# args = Args()
online = args.online
print(vars(args))
{'model_name': 'shd_1layer_large', 'dataset': 'shd', 'online': True, 'device': 'cuda', 'recurrency_type': 'none', 'lr': 0.0001, 'epochs': 1000, 'augment': True, 'batch_size': 128, 'n_hidden': [1332], 'inp_thr': 0.05, 'c_y': [1.5, -1.5], 'n_inputs': 700, 'n_outputs': 20, 'n_time_bins': 100, 'beta': 0.95}
Spiking Heidelberg Digits
#train_loader, test_loader = load_PMNIST(n_time_bins, scale=0.9, patches=True) #load_NMNIST(n_time_bins, batch_size=batch_size)
n_time_bins = 100
train_loader, test_loader = load_SHD(batch_size=batch_size) #load_NMNIST(n_time_bins, batch_size=batch_size)
# Plot Example(s)
for i in range(1):
frames, target = train_loader.next_item(-1, contrastive=True)
plt.figure(figsize=(10, 10))
plt.axis('off')
plt.imshow(frames.squeeze(1).T)
# plt.colorbar()
print(frames.shape, target)
plt.axis('on')
/home/lars/ownCloud/ETH/Master/Project_2/SNN_CLAPP/data.py:28: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor). self.y = torch.tensor(y)
torch.Size([100, 1, 700]) tensor([4.])
(-0.5, 99.5, 699.5, -0.5)
SNN = EchoSpike(args.n_inputs, args.n_hidden, beta=args.beta, c_y=args.c_y, device=device, recurrency_type=args.recurrency_type, online=args.online).to(device)
SNN.load_state_dict(torch.load(folder+model_name, map_location=device))
# train(SNN, train_loader, args.epochs, device, args.model_name,
# batch_size=args.batch_size, online=args.online, lr=1e-8, augment=args.augment)
from_epoch = 0
echo_train_loss = torch.load(folder+model_name[:-3]+'_loss_hist.pt', map_location='cpu')[int(from_epoch*len(train_loader)/args.batch_size):]
print(echo_train_loss.shape)
for i in range(echo_train_loss.shape[-1]):
plt.plot(from_epoch+(args.batch_size*np.arange(echo_train_loss.shape[0])/len(train_loader)), savgol_filter(echo_train_loss[:,i], 99, 1), color=color_list[i])
plt.legend([f'layer {i+1}' for i in range(len(SNN.layers))])
# no y ticks, because it's not really meaningful
plt.yticks([])
plt.xlabel('Epoch')
plt.ylabel('EchoSpike Loss')
torch.Size([63719, 1])
Text(0, 0.5, 'EchoSpike Loss')
# plotting adaptive threshold and update rate for an example
while True:
init_echo, label_0 = train_loader.next_item(-1, contrastive=True)
sample_1, label_1 = train_loader.next_item(-1, contrastive=True)
sample_2, label_2 = train_loader.next_item(label_1, contrastive=False)
if label_0 == 9 and label_1 == 7:
break
print(label_0, label_1, label_2)
SNN.eval()
with torch.no_grad():
# feed first sample to get initial activity
for t in range(100):
inp_activity = init_echo[t].mean(axis=-1)
SNN(init_echo[t], torch.tensor(-1, device=device), inp_activity=inp_activity)
SNN.reset(-1)
# feed second sample to get the update rates and thresholds for contrastive case
contrastive_thresholds = torch.zeros(100)
contrastive_temp_sim = torch.zeros((len(SNN.layers), 100))
for t in range(100):
inp_activity = sample_1[t].mean(axis=-1)
out_spk, mems, losses = SNN(sample_1[t], torch.tensor(-1, device=device), inp_activity=inp_activity)
contrastive_thresholds[t] = inp_activity * args.c_y[1]
contrastive_temp_sim[:, t] = losses
SNN.reset(-1)
# feed third sample to get the update rates and thresholds for predictive case
predictive_thresholds = torch.zeros(100)
predictive_temp_sim = torch.zeros((len(SNN.layers), 100))
for t in range(100):
inp_activity = sample_2[t].mean(axis=-1)
out_spk, mems, losses = SNN(sample_1[t], torch.tensor(1, device=device), inp_activity=inp_activity)
predictive_thresholds[t] = inp_activity * args.c_y[0]
predictive_temp_sim[:, t] = -losses
SNN.reset(1)
# plot thresholds, with sample as background
layer = 3
fig, ax = plt.subplots(figsize=(10, 5))
ax2 = ax.twinx()
# imshow in background
ax.imshow(sample_1.squeeze(1).T, aspect='auto', cmap='Reds')
ax2.plot(-contrastive_temp_sim[layer], color='r', label='Negative Similarity Score')
ax2.plot(contrastive_thresholds, color='r', linestyle='--', label='Adaptive Threshold')
ax2.hlines(args.inp_thr*args.c_y[1], 0, 100, color='r', linestyle=':', label='Input Threshold (times c(-1))')
# highlight regions where the thresholds are crossed
argwhere = np.argwhere(np.logical_and((-contrastive_temp_sim[layer] < contrastive_thresholds).numpy(), contrastive_thresholds.numpy() < args.inp_thr*args.c_y[1]))
for i in range(argwhere.shape[0]):
ax2.axvspan(argwhere[i], argwhere[i]+1, color='r', alpha=0.2, lw=0)
ax.yaxis.set_visible(False)
ax2.spines['right'].set_visible(False)
ax2.yaxis.tick_left()
ax2.yaxis.set_label_position('left')
ax2.set_xlim(ax.get_xlim())
# get rid of right margin
ax2.margins(0)
ax.set_xlabel('Timesteps')
plt.ylabel('Thresholds & Similarity Score')
plt.xlim(0, 100)
plt.legend()
# same for predictive
fig, ax = plt.subplots(figsize=(10, 5))
ax2 = ax.twinx()
# imshow in background
ax.imshow(sample_2.squeeze(1).T, aspect='auto', cmap='Blues')
ax2.plot(predictive_temp_sim[layer], color='b', label='Similarity Score')
ax2.plot(predictive_thresholds, color='b', linestyle='--', label='Adaptive Threshold')
ax2.hlines(args.inp_thr*args.c_y[0], 0, 100, color='b', linestyle=':', label='Input Threshold (times c(1))')
# highlight regions where the thresholds are crossed
argwhere = np.argwhere(np.logical_and((predictive_temp_sim[layer] < predictive_thresholds).numpy(), predictive_thresholds.numpy() > args.inp_thr*args.c_y[0]))
for i in range(argwhere.shape[0]):
ax2.axvspan(argwhere[i], argwhere[i]+1, color='b', alpha=0.1, lw=0)
ax.yaxis.set_visible(False)
ax2.spines['right'].set_visible(False)
ax2.yaxis.tick_left()
ax2.yaxis.set_label_position('left')
ax2.set_xlim(ax.get_xlim())
# get rid of right margin
#ax2.margins(0)
ax.set_xlabel('Timesteps')
plt.ylabel('Thresholds & Similarity Score')
plt.xlim(0, 100)
plt.legend()
plt.show()
tensor([9.]) tensor([7.]) tensor([7.])
--------------------------------------------------------------------------- IndexError Traceback (most recent call last) Input In [4], in <cell line: 10>() 38 # imshow in background 39 ax.imshow(sample_1.squeeze(1).T, aspect='auto', cmap='Reds') ---> 40 ax2.plot(-contrastive_temp_sim[layer], color='r', label='Negative Similarity Score') 41 ax2.plot(contrastive_thresholds, color='r', linestyle='--', label='Adaptive Threshold') 42 ax2.hlines(args.inp_thr*args.c_y[1], 0, 100, color='r', linestyle=':', label='Input Threshold (times c(-1))') IndexError: index 3 is out of bounds for dimension 0 with size 1
layers = [SNN.layers[0].fc.weight[:,:args.n_inputs]]
for i in range(1, len(SNN.layers)):
layers.append(SNN.layers[i].fc.weight[:,:args.n_hidden[i-1]] @ layers[-1])
for i in range(len(SNN.layers)):
plt.figure()
plt.imshow(SNN.layers[i].fc.weight.detach(), cmap='viridis')
plt.colorbar()
# plt.figure()
# plt.imshow(SNN.layers[i].pred.weight.detach(), vmax=0.5, vmin=-0.5)
# plt.colorbar()
for lay in layers:
plt.figure()
plt.imshow(lay.detach())
plt.colorbar()
from tqdm.notebook import tqdm
def train_out_proj(epochs, batch, cat, out_projs=None):
# train output projections from all layers (and no layer)
losses_out = []
beta = 1.0
lr = 1e-4
augment = True
optimizers = []
print_interval = 10*batch
if out_projs is None:
out_projs = []
out_proj_0 = simple_out(700, 20, beta=beta)
else:
for out_p in out_projs:
out_p.train()
out_p.reset()
out_proj_0 = out_projs[0]
out_projs = out_projs[1:]
optim_0 = torch.optim.Adam(out_proj_0.parameters(), lr=lr)
for lay in range(len(SNN.layers)):
if len(out_projs) <= lay:
if cat:
out_projs.append(simple_out(sum(args.n_hidden[:lay+1])+700, 20, beta=beta))
else:
out_projs.append(simple_out(args.n_hidden[lay], 20, beta=beta))
optimizers.append(torch.optim.Adam(out_projs[lay].parameters(), lr=lr))
optimizers[-1].zero_grad()
SNN.eval()
acc = []
target = batch_size*[-1]
correct = (len(SNN.layers) + 1)*[0]
with torch.no_grad():
pbar = tqdm(total=len(train_loader)*epochs)
while len(losses_out)*batch < len(train_loader)*epochs:
data, target = train_loader.next_item(target, contrastive=True)
SNN.reset(0)
logit_lists = [[] for _ in range(len(SNN.layers)+1)]
data = data.squeeze()
if augment:
data = augment_shd(data)
for step in range(data.shape[0]):
data_step = data[step].float().to(device)
target = target.to(device)
logits, _, _ = SNN(data_step, 0)
if step == args.n_time_bins-1:
_, logts = out_proj_0(data_step, target)
logit_lists[0] = logts
for lay in range(len(SNN.layers)):
if cat:
data_step = torch.cat([data_step, logits[lay]], dim=-1)
else:
data_step = logits[lay]
_, logts = out_projs[lay](data_step, target)
logit_lists[lay+1] = logts
else:
out_proj_0(data_step, None)
for lay in range(len(SNN.layers)):
if cat:
data_step = torch.cat([data_step, logits[lay]], dim=-1)
else:
data_step = logits[lay]
out_projs[lay](data_step, None)
preds = [logit_lists[lay].argmax(axis=-1) for lay in range(len(SNN.layers)+1)]
correct = [correct[lay] + (preds[lay] == target).sum() for lay in range(len(SNN.layers)+1)]
out_proj_0.reset()
for i, out_proj in enumerate(out_projs):
out_proj.reset()
losses_out.append(torch.tensor([torch.nn.functional.cross_entropy(logit_lists[lay], target.squeeze().long()) for lay in range(len(SNN.layers)+1)], requires_grad=False))
optim_0.step()
optim_0.zero_grad()
for opt in optimizers:
opt.step()
opt.zero_grad()
if len(losses_out)*batch % print_interval == 0:
pbar.write(f'Cross Entropy Loss: {(torch.stack(losses_out)[-print_interval//batch:].sum(dim=0)/(print_interval//batch)).numpy()}\n' +
f'Correct: {100*np.array(correct)/print_interval}%')
acc.append(np.array(correct)/print_interval)
correct = (len(SNN.layers) + 1)*[0]
pbar.update(batch)
return [out_proj_0, *out_projs], np.asarray(acc), torch.stack(losses_out)
with torch.no_grad():
if args.augment:
n_epochs = 100
cat = True
# if already trained, load the output projections
if os.path.exists(folder+model_name[:-3]+'_out_projs.pt') and False:
out_projs = torch.load(folder+model_name[:-3]+'_out_projs.pt', map_location=device)
else:
out_projs, acc, losses_out = train_out_proj(n_epochs, batch_size, cat)
test_acc = get_accuracy(SNN, out_projs, test_loader, device, cat, )
train_acc = get_accuracy(SNN, out_projs, train_loader, device, cat)
torch.save(out_projs, folder+model_name[:-3]+'_out_projs.pt')
0%| | 0/815600 [00:00<?, ?it/s]
Cross Entropy Loss: [14.313995 9.099342] Correct: [5.625 5.859375]% Cross Entropy Loss: [8.187934 6.3089476] Correct: [4.921875 6.484375]% Cross Entropy Loss: [6.951187 5.181357] Correct: [ 4.53125 10.078125]% Cross Entropy Loss: [5.946702 4.473062] Correct: [ 7.890625 10.234375]% Cross Entropy Loss: [5.4507284 3.974528 ] Correct: [ 7.265625 13.046875]% Cross Entropy Loss: [4.881641 3.6358018] Correct: [ 8.828125 14.6875 ]% Cross Entropy Loss: [4.61839 3.4733715] Correct: [ 9.84375 18.203125]% Cross Entropy Loss: [4.3676906 3.2100844] Correct: [ 9.0625 17.03125]% Cross Entropy Loss: [4.2846184 3.0249293] Correct: [10.78125 23.28125]% Cross Entropy Loss: [3.8945746 2.8896012] Correct: [12.1875 23.125 ]% Cross Entropy Loss: [3.820954 2.9955711] Correct: [10.546875 23.59375 ]% Cross Entropy Loss: [3.564299 2.6497583] Correct: [13.75 24.765625]% Cross Entropy Loss: [3.6856475 2.7007663] Correct: [14.609375 23.203125]% Cross Entropy Loss: [3.4151406 2.6983943] Correct: [13.828125 25.234375]% Cross Entropy Loss: [3.4241405 2.6245522] Correct: [14.0625 26.09375]% Cross Entropy Loss: [3.3426914 2.5967703] Correct: [15.46875 26.484375]% Cross Entropy Loss: [3.2667968 2.6326337] Correct: [17.265625 27.890625]% Cross Entropy Loss: [3.320641 2.6780038] Correct: [16.25 27.109375]% Cross Entropy Loss: [3.169768 2.4643428] Correct: [16.09375 29.84375]% Cross Entropy Loss: [3.1465955 2.3945916] Correct: [15.546875 30.15625 ]% Cross Entropy Loss: [2.986266 2.3093007] Correct: [18.59375 30.3125 ]% Cross Entropy Loss: [3.0388336 2.4310157] Correct: [20.234375 31.171875]% Cross Entropy Loss: [3.0892272 2.3056035] Correct: [19.6875 32.03125]% Cross Entropy Loss: [3.0571368 2.421267 ] Correct: [17.578125 29.84375 ]% Cross Entropy Loss: [3.0852628 2.3704216] Correct: [19.375 30.390625]% Cross Entropy Loss: [3.0512128 2.3868928] Correct: [19.296875 32.109375]% Cross Entropy Loss: [3.027636 2.3340905] Correct: [19.609375 32.65625 ]% Cross Entropy Loss: [3.0781162 2.2549465] Correct: [20.390625 34.84375 ]% Cross Entropy Loss: [2.893828 2.2259786] Correct: [20.546875 34.21875 ]% Cross Entropy Loss: [2.9055302 2.2219296] Correct: [21.875 35.15625]% Cross Entropy Loss: [2.8676295 2.108374 ] Correct: [21.328125 35.703125]% Cross Entropy Loss: [2.8262253 2.1915321] Correct: [20.859375 33.59375 ]% Cross Entropy Loss: [2.8070726 2.0325317] Correct: [22.34375 37.1875 ]% Cross Entropy Loss: [2.7413309 1.9289356] Correct: [23.515625 39.6875 ]% Cross Entropy Loss: [2.8622472 2.2309709] Correct: [21.5625 35.390625]% Cross Entropy Loss: [2.8631537 2.1316493] Correct: [25.546875 36.015625]% Cross Entropy Loss: [2.8161116 2.2107635] Correct: [21.328125 34.375 ]% Cross Entropy Loss: [2.9467907 2.1844616] Correct: [21.875 35.390625]% Cross Entropy Loss: [2.8130527 2.113578 ] Correct: [21.015625 35.9375 ]% Cross Entropy Loss: [2.747198 2.089418] Correct: [22.734375 37.34375 ]% Cross Entropy Loss: [2.6926951 2.0189242] Correct: [23.59375 38.203125]% Cross Entropy Loss: [2.6316378 1.8943977] Correct: [26.09375 40.390625]% Cross Entropy Loss: [2.737908 1.9645641] Correct: [24.0625 38.828125]% Cross Entropy Loss: [2.8403568 2.0318146] Correct: [22.65625 39.84375]% Cross Entropy Loss: [2.7549195 1.822452 ] Correct: [24.0625 43.75 ]% Cross Entropy Loss: [2.6795113 2.0256016] Correct: [21.953125 37.109375]% Cross Entropy Loss: [2.5867848 1.7345188] Correct: [27.1875 45.078125]% Cross Entropy Loss: [2.6175826 1.8836162] Correct: [25.625 41.09375]% Cross Entropy Loss: [2.7350922 1.8651159] Correct: [22.34375 43.515625]% Cross Entropy Loss: [2.6211905 1.7962688] Correct: [27.109375 43.828125]% Cross Entropy Loss: [2.5825527 1.8827003] Correct: [25.46875 39.6875 ]% Cross Entropy Loss: [2.6376565 1.7836764] Correct: [25.9375 43.75 ]% Cross Entropy Loss: [2.5825844 1.7944248] Correct: [24.765625 42.03125 ]% Cross Entropy Loss: [2.571115 1.7913485] Correct: [26.875 41.015625]% Cross Entropy Loss: [2.5827973 1.9166778] Correct: [27.578125 40.546875]% Cross Entropy Loss: [2.6336613 1.8473384] Correct: [25.546875 43.515625]% Cross Entropy Loss: [2.5386684 1.7500595] Correct: [28.671875 44.84375 ]% Cross Entropy Loss: [2.6029117 1.8635193] Correct: [27.03125 42.96875]% Cross Entropy Loss: [2.6544895 1.8170935] Correct: [25. 43.828125]% Cross Entropy Loss: [2.5716615 1.7597157] Correct: [25.703125 44.609375]% Cross Entropy Loss: [2.5358777 1.785189 ] Correct: [26.171875 44.21875 ]% Cross Entropy Loss: [2.45581 1.7592008] Correct: [26.09375 46.09375]% Cross Entropy Loss: [2.6317296 1.8432165] Correct: [23.59375 42.578125]% Cross Entropy Loss: [2.477721 1.562176] Correct: [27.5 50.859375]% Cross Entropy Loss: [2.5966911 1.6501175] Correct: [26.71875 47.890625]% Cross Entropy Loss: [2.6283836 1.8342499] Correct: [26.171875 44.296875]% Cross Entropy Loss: [2.5891643 1.8224297] Correct: [27.109375 45. ]% Cross Entropy Loss: [2.5839498 1.770544 ] Correct: [26.640625 45.625 ]% Cross Entropy Loss: [2.5813203 1.7426443] Correct: [25.15625 44.0625 ]% Cross Entropy Loss: [2.512799 1.6958224] Correct: [28.28125 45.546875]% Cross Entropy Loss: [2.522997 1.7454729] Correct: [27.03125 46.171875]% Cross Entropy Loss: [2.5243523 1.7465271] Correct: [26.640625 46.171875]% Cross Entropy Loss: [2.4010887 1.5776136] Correct: [29.921875 48.515625]% Cross Entropy Loss: [2.4520717 1.6431043] Correct: [29.53125 47.03125]% Cross Entropy Loss: [2.5002449 1.62231 ] Correct: [28.046875 46.796875]% Cross Entropy Loss: [2.448183 1.6866318] Correct: [27.265625 47.34375 ]% Cross Entropy Loss: [2.3278177 1.5603622] Correct: [31.796875 48.125 ]% Cross Entropy Loss: [2.502673 1.572726] Correct: [28.984375 48.75 ]% Cross Entropy Loss: [2.460004 1.5865178] Correct: [29.21875 48.125 ]% Cross Entropy Loss: [2.415388 1.6269791] Correct: [30. 48.046875]% Cross Entropy Loss: [2.5293667 1.6809063] Correct: [26.875 47.578125]% Cross Entropy Loss: [2.374123 1.503344] Correct: [30.234375 50.9375 ]% Cross Entropy Loss: [2.382563 1.6600912] Correct: [27.8125 46.796875]% Cross Entropy Loss: [2.330771 1.4367571] Correct: [27.96875 52.734375]% Cross Entropy Loss: [2.2551608 1.4061832] Correct: [32.03125 52.1875 ]% Cross Entropy Loss: [2.4431841 1.6001146] Correct: [29.765625 48.359375]% Cross Entropy Loss: [2.3067389 1.4722195] Correct: [32.578125 52.96875 ]% Cross Entropy Loss: [2.4242828 1.5623109] Correct: [30.546875 51.015625]% Cross Entropy Loss: [2.404963 1.5988398] Correct: [30.46875 49.375 ]% Cross Entropy Loss: [2.3738644 1.6192579] Correct: [31.25 50.46875]% Cross Entropy Loss: [2.2819183 1.5332501] Correct: [31.40625 50.234375]% Cross Entropy Loss: [2.4128027 1.5696067] Correct: [28.59375 50.234375]% Cross Entropy Loss: [2.5017002 1.609463 ] Correct: [27.578125 48.125 ]% Cross Entropy Loss: [2.5566263 1.6966852] Correct: [28.75 46.015625]% Cross Entropy Loss: [2.5410113 1.6571732] Correct: [29.21875 48.984375]% Cross Entropy Loss: [2.3229337 1.4808272] Correct: [28.28125 50.390625]% Cross Entropy Loss: [2.4184113 1.5954044] Correct: [28.90625 48.59375]% Cross Entropy Loss: [2.567089 1.6687448] Correct: [26.40625 46.953125]% Cross Entropy Loss: [2.3976378 1.5823095] Correct: [29.21875 49.6875 ]% Cross Entropy Loss: [2.4813595 1.5941111] Correct: [26.25 47.34375]% Cross Entropy Loss: [2.3416488 1.4831976] Correct: [29.375 49.0625]% Cross Entropy Loss: [2.33704 1.40119] Correct: [30.546875 55.703125]% Cross Entropy Loss: [2.4844353 1.5852791] Correct: [25.546875 50.546875]% Cross Entropy Loss: [2.25913 1.4476125] Correct: [32.578125 51.71875 ]% Cross Entropy Loss: [2.3925385 1.5014861] Correct: [31.40625 51.40625]% Cross Entropy Loss: [2.3728828 1.5486963] Correct: [30. 51.25]% Cross Entropy Loss: [2.396335 1.5730026] Correct: [30.703125 48.28125 ]% Cross Entropy Loss: [2.315197 1.4658949] Correct: [30.15625 51.953125]% Cross Entropy Loss: [2.2947965 1.476065 ] Correct: [29.921875 52.578125]% Cross Entropy Loss: [2.3443992 1.459317 ] Correct: [31.640625 51.796875]% Cross Entropy Loss: [2.3687196 1.5258805] Correct: [29.453125 50.859375]% Cross Entropy Loss: [2.2844274 1.4674822] Correct: [31.015625 52.65625 ]% Cross Entropy Loss: [2.2579598 1.5650508] Correct: [33.75 51.5625]% Cross Entropy Loss: [2.22128 1.3795776] Correct: [30.859375 54.21875 ]% Cross Entropy Loss: [2.2676878 1.4663472] Correct: [32.1875 52.65625]% Cross Entropy Loss: [2.3793247 1.5582575] Correct: [28.046875 50.546875]% Cross Entropy Loss: [2.2854948 1.4375732] Correct: [32.578125 53.046875]% Cross Entropy Loss: [2.2691517 1.3015031] Correct: [31.328125 54.609375]% Cross Entropy Loss: [2.3235888 1.4401485] Correct: [32.265625 53.203125]% Cross Entropy Loss: [2.2807784 1.3761926] Correct: [29.375 55.234375]% Cross Entropy Loss: [2.2766078 1.3260014] Correct: [31.25 56.015625]% Cross Entropy Loss: [2.3384786 1.3972623] Correct: [30.703125 54.0625 ]% Cross Entropy Loss: [2.239877 1.3276863] Correct: [33.359375 56.25 ]% Cross Entropy Loss: [2.3611073 1.3327819] Correct: [29.6875 56.5625]% Cross Entropy Loss: [2.261144 1.3886083] Correct: [32.265625 54.53125 ]% Cross Entropy Loss: [2.3639069 1.4025548] Correct: [31.953125 53.203125]% Cross Entropy Loss: [2.3807318 1.4964199] Correct: [30.46875 50.859375]% Cross Entropy Loss: [2.2797737 1.4531746] Correct: [32.65625 52.34375]% Cross Entropy Loss: [2.3015342 1.3775599] Correct: [30.859375 55.3125 ]% Cross Entropy Loss: [2.4187794 1.499069 ] Correct: [29.375 50.859375]% Cross Entropy Loss: [2.3531373 1.4245743] Correct: [31.71875 53.4375 ]% Cross Entropy Loss: [2.3349235 1.4023972] Correct: [31.875 53.28125]% Cross Entropy Loss: [2.357822 1.438986] Correct: [29.6875 52.1875]% Cross Entropy Loss: [2.400087 1.4807303] Correct: [30.625 49.6875]% Cross Entropy Loss: [2.3477216 1.4440556] Correct: [31.875 53.515625]% Cross Entropy Loss: [2.2957122 1.3509364] Correct: [31.171875 54.21875 ]% Cross Entropy Loss: [2.271256 1.2873652] Correct: [33.59375 57.34375]% Cross Entropy Loss: [2.2768269 1.2799364] Correct: [32.265625 57.65625 ]% Cross Entropy Loss: [2.3255336 1.380134 ] Correct: [32.109375 55.3125 ]% Cross Entropy Loss: [2.2882273 1.4125729] Correct: [32.421875 53.28125 ]% Cross Entropy Loss: [2.398233 1.4821151] Correct: [28.28125 52.890625]% Cross Entropy Loss: [2.2781453 1.3212835] Correct: [30.3125 56.171875]% Cross Entropy Loss: [2.2919357 1.3750699] Correct: [31.5625 54.453125]% Cross Entropy Loss: [2.20497 1.2765039] Correct: [35.3125 57.578125]% Cross Entropy Loss: [2.272082 1.3382531] Correct: [32.578125 55.546875]% Cross Entropy Loss: [2.2032666 1.2599838] Correct: [33.984375 57.34375 ]% Cross Entropy Loss: [2.2773478 1.2810342] Correct: [32.890625 56.328125]% Cross Entropy Loss: [2.3670692 1.3336469] Correct: [29.6875 56.09375]% Cross Entropy Loss: [2.4782019 1.5009553] Correct: [28.75 53.75]% Cross Entropy Loss: [2.2481072 1.3238726] Correct: [33.671875 57.03125 ]% Cross Entropy Loss: [2.3733873 1.476785 ] Correct: [31.328125 51.953125]% Cross Entropy Loss: [2.299612 1.3109412] Correct: [28.828125 57.03125 ]% Cross Entropy Loss: [2.40865 1.4642934] Correct: [30.546875 52.421875]% Cross Entropy Loss: [2.2160816 1.3334074] Correct: [32.03125 56.796875]% Cross Entropy Loss: [2.1265419 1.1166967] Correct: [31.875 60.9375]% Cross Entropy Loss: [2.1095867 1.275995 ] Correct: [35.15625 57.65625]% Cross Entropy Loss: [2.3229258 1.3776207] Correct: [32.65625 56.796875]% Cross Entropy Loss: [2.2739053 1.2817076] Correct: [30.703125 58.046875]% Cross Entropy Loss: [2.167472 1.2050197] Correct: [32.34375 60.078125]% Cross Entropy Loss: [2.2191257 1.2010783] Correct: [30.9375 59.296875]% Cross Entropy Loss: [2.2528615 1.3258622] Correct: [33.125 57.03125]% Cross Entropy Loss: [2.389148 1.3039352] Correct: [28.515625 55.78125 ]% Cross Entropy Loss: [2.3398554 1.3570752] Correct: [31.640625 54.6875 ]% Cross Entropy Loss: [2.3871522 1.424741 ] Correct: [31.09375 54.296875]% Cross Entropy Loss: [2.40531 1.4805454] Correct: [29.84375 51.71875]% Cross Entropy Loss: [2.2485678 1.3561897] Correct: [30.546875 56.5625 ]% Cross Entropy Loss: [2.516479 1.5137602] Correct: [28.75 51.875]% Cross Entropy Loss: [2.40506 1.4596338] Correct: [30.234375 52.421875]% Cross Entropy Loss: [2.12909 1.3146592] Correct: [33.90625 57.734375]% Cross Entropy Loss: [2.3817828 1.3444784] Correct: [29.609375 54.0625 ]% Cross Entropy Loss: [2.226515 1.3272054] Correct: [32.65625 55.546875]% Cross Entropy Loss: [2.2908902 1.3386853] Correct: [30.703125 55.625 ]% Cross Entropy Loss: [2.245748 1.2963207] Correct: [32.8125 57.5 ]% Cross Entropy Loss: [2.1577973 1.2150828] Correct: [34.296875 60.859375]% Cross Entropy Loss: [2.197129 1.2898552] Correct: [32.5 56.484375]% Cross Entropy Loss: [2.1791012 1.2298634] Correct: [33.828125 59.609375]% Cross Entropy Loss: [2.2695413 1.2715654] Correct: [31.796875 56.171875]% Cross Entropy Loss: [2.1917355 1.2653673] Correct: [35.3125 59.375 ]% Cross Entropy Loss: [2.1806483 1.3399513] Correct: [31.40625 57.421875]% Cross Entropy Loss: [2.2278328 1.2639043] Correct: [34.21875 57.578125]% Cross Entropy Loss: [2.1299484 1.2346227] Correct: [35.15625 59.0625 ]% Cross Entropy Loss: [2.2349045 1.306574 ] Correct: [32.34375 59.6875 ]% Cross Entropy Loss: [2.2293546 1.296579 ] Correct: [32.34375 56.484375]% Cross Entropy Loss: [2.0019922 1.1064491] Correct: [37.96875 63.671875]% Cross Entropy Loss: [2.1467674 1.1199954] Correct: [34.609375 62.8125 ]% Cross Entropy Loss: [2.314003 1.45438 ] Correct: [29.921875 51.640625]% Cross Entropy Loss: [2.2204108 1.3193465] Correct: [30.9375 57.5 ]% Cross Entropy Loss: [2.2615457 1.260369 ] Correct: [32.1875 56.71875]% Cross Entropy Loss: [2.215611 1.236027] Correct: [31.796875 59.453125]% Cross Entropy Loss: [2.2558377 1.2421529] Correct: [30.3125 57.1875]% Cross Entropy Loss: [2.2041094 1.3359329] Correct: [31.25 54.53125]% Cross Entropy Loss: [2.1226146 1.1927675] Correct: [35.859375 60.390625]% Cross Entropy Loss: [2.0108964 1.0700333] Correct: [36.71875 62.5 ]% Cross Entropy Loss: [2.1650405 1.2488431] Correct: [33.828125 58.984375]% Cross Entropy Loss: [2.0763164 1.159961 ] Correct: [33.984375 59.609375]% Cross Entropy Loss: [2.2887151 1.3284991] Correct: [32.421875 58.125 ]% Cross Entropy Loss: [2.2767625 1.2749548] Correct: [32.34375 57.96875]% Cross Entropy Loss: [2.1088898 1.0527987] Correct: [34.453125 64.453125]% Cross Entropy Loss: [2.2221723 1.2410733] Correct: [32.890625 58.515625]% Cross Entropy Loss: [2.2018695 1.1777635] Correct: [33.203125 61.015625]% Cross Entropy Loss: [2.3442333 1.3075377] Correct: [32.03125 55.78125]% Cross Entropy Loss: [2.2214828 1.2147939] Correct: [32.5 60.859375]% Cross Entropy Loss: [2.1835735 1.2677255] Correct: [32.890625 57.265625]% Cross Entropy Loss: [1.995806 1.0640777] Correct: [36.40625 63.59375]% Cross Entropy Loss: [2.243962 1.3389227] Correct: [33.203125 57.96875 ]% Cross Entropy Loss: [2.213465 1.2385209] Correct: [32.265625 59.375 ]% Cross Entropy Loss: [2.1252456 1.1576301] Correct: [33.828125 61.09375 ]% Cross Entropy Loss: [2.1604712 1.2302692] Correct: [34.21875 60.3125 ]% Cross Entropy Loss: [2.0066557 1.0711169] Correct: [36.71875 62.03125]% Cross Entropy Loss: [2.224689 1.2388389] Correct: [30.9375 58.59375]% Cross Entropy Loss: [2.0441384 1.0921986] Correct: [36.09375 63.4375 ]% Cross Entropy Loss: [2.1355634 1.1852753] Correct: [32.5 60.234375]% Cross Entropy Loss: [2.1778278 1.2086099] Correct: [33.515625 57.734375]% Cross Entropy Loss: [2.2058349 1.1783116] Correct: [30.78125 60.625 ]% Cross Entropy Loss: [2.2691238 1.1900282] Correct: [30.703125 60.078125]% Cross Entropy Loss: [2.1808307 1.1544319] Correct: [32.890625 62.5 ]% Cross Entropy Loss: [2.2770205 1.2187328] Correct: [32.890625 58.671875]% Cross Entropy Loss: [2.0733552 1.1033981] Correct: [33.90625 62.1875 ]% Cross Entropy Loss: [2.2562535 1.2965081] Correct: [30.9375 58.59375]% Cross Entropy Loss: [2.1257691 1.1786801] Correct: [36.015625 60.390625]% Cross Entropy Loss: [2.186401 1.2037128] Correct: [32.5 60. ]% Cross Entropy Loss: [2.1462145 1.2054521] Correct: [33.4375 60.46875]% Cross Entropy Loss: [2.3487113 1.2582158] Correct: [29.609375 58.671875]% Cross Entropy Loss: [2.143238 1.1597302] Correct: [35.15625 61.796875]% Cross Entropy Loss: [2.1555495 1.1899214] Correct: [33.046875 60.78125 ]% Cross Entropy Loss: [2.3535953 1.3144853] Correct: [32.578125 57.109375]% Cross Entropy Loss: [1.976383 1.0317686] Correct: [37.96875 65.3125 ]% Cross Entropy Loss: [2.1781986 1.133721 ] Correct: [33.515625 62.5 ]% Cross Entropy Loss: [2.133198 1.1597993] Correct: [34.53125 60.078125]% Cross Entropy Loss: [2.1323743 1.196222 ] Correct: [33.90625 62.265625]% Cross Entropy Loss: [2.2972589 1.2847755] Correct: [32.5 57.421875]% Cross Entropy Loss: [2.1190689 1.1783539] Correct: [32.734375 59.453125]% Cross Entropy Loss: [2.2093656 1.2345682] Correct: [33.359375 59.375 ]% Cross Entropy Loss: [2.1435971 1.11424 ] Correct: [33.828125 63.359375]% Cross Entropy Loss: [2.1122904 1.1555637] Correct: [35.703125 62.1875 ]% Cross Entropy Loss: [2.1720138 1.1783694] Correct: [32.578125 62.109375]% Cross Entropy Loss: [2.1315763 1.1422364] Correct: [33.125 61.796875]% Cross Entropy Loss: [1.9625031 0.99608314] Correct: [37.1875 64.6875]% Cross Entropy Loss: [2.2433178 1.1916873] Correct: [33.515625 60.78125 ]% Cross Entropy Loss: [2.2122304 1.14859 ] Correct: [33.4375 61.953125]% Cross Entropy Loss: [2.0847392 1.0998975] Correct: [34.6875 61.5625]% Cross Entropy Loss: [2.111541 1.1216688] Correct: [35. 60.546875]% Cross Entropy Loss: [2.0939105 1.0984247] Correct: [35.46875 62.578125]% Cross Entropy Loss: [2.0840175 1.0733654] Correct: [33.515625 61.953125]% Cross Entropy Loss: [2.059178 1.0151303] Correct: [35. 65.390625]% Cross Entropy Loss: [2.1241944 1.1936681] Correct: [33.984375 61.5625 ]% Cross Entropy Loss: [2.152366 1.1261685] Correct: [34.21875 62.1875 ]% Cross Entropy Loss: [2.2062345 1.1785904] Correct: [33.203125 60.234375]% Cross Entropy Loss: [2.0634491 1.0513628] Correct: [35.15625 65.78125]% Cross Entropy Loss: [2.0277543 1.1403968] Correct: [36.171875 62.03125 ]% Cross Entropy Loss: [2.1327224 1.112538 ] Correct: [33.4375 63.125 ]% Cross Entropy Loss: [1.9441202 0.9338592] Correct: [35.859375 67.265625]% Cross Entropy Loss: [2.14573 1.1241181] Correct: [35.15625 63.59375]% Cross Entropy Loss: [2.058078 1.0698426] Correct: [34.140625 63.046875]% Cross Entropy Loss: [2.1855145 1.1218735] Correct: [34.609375 62.109375]% Cross Entropy Loss: [1.9065931 0.9865529] Correct: [38.59375 65.859375]% Cross Entropy Loss: [2.0935419 1.0306355] Correct: [34.921875 65.234375]% Cross Entropy Loss: [2.1347167 1.1488874] Correct: [32.34375 62.578125]% Cross Entropy Loss: [2.25144 1.2165654] Correct: [33.828125 59.765625]% Cross Entropy Loss: [2.1221347 1.1931523] Correct: [32.96875 60.234375]% Cross Entropy Loss: [1.9877834 1.0697641] Correct: [38.359375 63.203125]% Cross Entropy Loss: [1.901036 0.9767046] Correct: [40.78125 69.140625]% Cross Entropy Loss: [2.017328 0.9839282] Correct: [37.03125 69.0625 ]% Cross Entropy Loss: [2.0206945 1.0241355] Correct: [35.390625 67.265625]% Cross Entropy Loss: [2.0634017 1.0314963] Correct: [34.6875 63.515625]% Cross Entropy Loss: [2.0637345 1.1136535] Correct: [34.6875 61.25 ]% Cross Entropy Loss: [2.165344 1.1659467] Correct: [34.21875 60.546875]% Cross Entropy Loss: [1.9341033 0.98996276] Correct: [38.203125 68.046875]% Cross Entropy Loss: [2.0767736 1.1296029] Correct: [36.015625 62.96875 ]% Cross Entropy Loss: [1.9799122 0.9830142] Correct: [37.5 68.046875]% Cross Entropy Loss: [2.1288888 1.102431 ] Correct: [35.546875 63.515625]% Cross Entropy Loss: [2.1784267 1.2206053] Correct: [34.53125 60.703125]% Cross Entropy Loss: [1.9744213 0.98222274] Correct: [37.96875 67.1875 ]% Cross Entropy Loss: [2.1761372 1.214867 ] Correct: [31.5625 57.578125]% Cross Entropy Loss: [2.1439831 1.0546397] Correct: [32.265625 65. ]% Cross Entropy Loss: [2.3111758 1.2379646] Correct: [32.265625 58.828125]% Cross Entropy Loss: [1.918314 0.9939888] Correct: [39.140625 64.765625]% Cross Entropy Loss: [2.1154976 1.0898044] Correct: [33.359375 63.59375 ]% Cross Entropy Loss: [2.2802463 1.2327421] Correct: [32.421875 58.75 ]% Cross Entropy Loss: [1.9258674 0.96821374] Correct: [39.453125 65.859375]% Cross Entropy Loss: [1.9349041 0.9977549] Correct: [38.90625 65.390625]% Cross Entropy Loss: [1.9564587 0.9257614] Correct: [39.0625 68.671875]% Cross Entropy Loss: [2.0662205 0.9779871] Correct: [36.5625 68.828125]% Cross Entropy Loss: [2.283265 1.1285784] Correct: [32.5 62.96875]% Cross Entropy Loss: [1.9384588 0.9834124] Correct: [38.90625 66.015625]% Cross Entropy Loss: [2.0634747 1.0879116] Correct: [33.828125 63.046875]% Cross Entropy Loss: [2.0325809 0.97033894] Correct: [36.953125 67.421875]% Cross Entropy Loss: [2.129327 1.1245673] Correct: [34.84375 62.96875]% Cross Entropy Loss: [2.2052293 1.1474326] Correct: [32.03125 61.171875]% Cross Entropy Loss: [2.0500762 0.9504854] Correct: [36.25 67.265625]% Cross Entropy Loss: [2.1011226 1.0427713] Correct: [35.390625 65.078125]% Cross Entropy Loss: [2.1309547 1.0412221] Correct: [35.3125 63.515625]% Cross Entropy Loss: [2.2090192 1.1062742] Correct: [32.421875 61.953125]% Cross Entropy Loss: [2.2421412 1.1895859] Correct: [31.640625 61.40625 ]% Cross Entropy Loss: [2.0257316 0.99122363] Correct: [36.25 66.71875]% Cross Entropy Loss: [2.14191 1.0704272] Correct: [33.359375 63.046875]% Cross Entropy Loss: [2.0045218 0.9369028] Correct: [37.734375 67.265625]% Cross Entropy Loss: [2.160404 1.1237806] Correct: [33.4375 63.28125]% Cross Entropy Loss: [1.9392579 0.92556363] Correct: [37.8125 69.609375]% Cross Entropy Loss: [1.9910405 0.95980215] Correct: [37.421875 66.484375]% Cross Entropy Loss: [2.1098943 1.0629327] Correct: [34.53125 65.46875]% Cross Entropy Loss: [1.9656417 1.0099294] Correct: [36.09375 65.625 ]% Cross Entropy Loss: [2.120175 1.0691597] Correct: [37.1875 63.984375]% Cross Entropy Loss: [1.9666752 1.1107584] Correct: [40.15625 64.921875]% Cross Entropy Loss: [2.1508212 1.1373085] Correct: [34.453125 62.890625]% Cross Entropy Loss: [2.0738356 1.0416645] Correct: [36.171875 64.375 ]% Cross Entropy Loss: [1.9455401 1.0102122] Correct: [36.328125 66.796875]% Cross Entropy Loss: [2.0634446 1.0516846] Correct: [36.71875 63.515625]% Cross Entropy Loss: [2.2197862 1.080001 ] Correct: [32.8125 64.53125]% Cross Entropy Loss: [2.233566 1.1801441] Correct: [33.046875 60.9375 ]% Cross Entropy Loss: [1.9392802 0.88568294] Correct: [38.046875 68.828125]% Cross Entropy Loss: [1.9014928 0.8435947] Correct: [38.90625 70.625 ]% Cross Entropy Loss: [1.88759 0.90144455] Correct: [38.59375 70.703125]% Cross Entropy Loss: [2.0626452 0.99576885] Correct: [35.078125 65.46875 ]% Cross Entropy Loss: [2.1470504 1.1259902] Correct: [36.09375 62.109375]% Cross Entropy Loss: [1.9646857 0.89839536] Correct: [37.734375 69.453125]% Cross Entropy Loss: [2.0850308 0.98927116] Correct: [34.296875 66.25 ]% Cross Entropy Loss: [1.9850029 0.9412414] Correct: [38.203125 68.046875]% Cross Entropy Loss: [1.9710951 1.0479825] Correct: [37.65625 64.921875]% Cross Entropy Loss: [1.9712403 0.8848939] Correct: [37.65625 70.390625]% Cross Entropy Loss: [2.0818489 1.0339851] Correct: [35. 65.390625]% Cross Entropy Loss: [2.0616808 1.0331202] Correct: [36.953125 65.859375]% Cross Entropy Loss: [2.1676717 1.0887108] Correct: [33.984375 62.890625]% Cross Entropy Loss: [2.0967023 0.98823863] Correct: [34.140625 66.953125]% Cross Entropy Loss: [2.181618 1.1704443] Correct: [34.140625 61.640625]% Cross Entropy Loss: [1.9658992 0.92764837] Correct: [38.28125 69.453125]% Cross Entropy Loss: [2.0889847 1.0531651] Correct: [34.453125 65.546875]% Cross Entropy Loss: [1.9373844 0.91875184] Correct: [38.59375 68.59375]% Cross Entropy Loss: [1.8976494 0.8835932] Correct: [38.203125 69.296875]% Cross Entropy Loss: [2.1454206 1.0919864] Correct: [32.421875 62.890625]% Cross Entropy Loss: [1.9757704 1.0120105] Correct: [36.328125 63.515625]% Cross Entropy Loss: [2.114286 1.0862621] Correct: [35.234375 65.625 ]% Cross Entropy Loss: [1.9755837 0.94148743] Correct: [35.859375 68.4375 ]% Cross Entropy Loss: [2.131722 1.0045172] Correct: [36.171875 65.703125]% Cross Entropy Loss: [1.9944563 0.9981648] Correct: [37.109375 65.546875]% Cross Entropy Loss: [2.0344603 0.98477805] Correct: [36.796875 67.03125 ]% Cross Entropy Loss: [2.059626 0.9630005] Correct: [37.5 67.578125]% Cross Entropy Loss: [1.965571 0.95037633] Correct: [37.890625 68.671875]% Cross Entropy Loss: [2.114256 1.041924] Correct: [34.921875 64.453125]% Cross Entropy Loss: [2.0621521 1.0096956] Correct: [36.71875 66.171875]% Cross Entropy Loss: [1.9320924 0.97648 ] Correct: [37.890625 66.953125]% Cross Entropy Loss: [1.9095819 0.929115 ] Correct: [37.421875 67.265625]% Cross Entropy Loss: [1.9487722 0.8894169] Correct: [37.109375 70.46875 ]% Cross Entropy Loss: [1.8487208 0.7841818] Correct: [42.421875 74.21875 ]% Cross Entropy Loss: [1.9685814 0.92524815] Correct: [37.65625 68.828125]% Cross Entropy Loss: [2.0319257 0.98477745] Correct: [37.578125 67.734375]% Cross Entropy Loss: [1.9917917 0.938358 ] Correct: [37.65625 67.265625]% Cross Entropy Loss: [1.9789757 0.88956416] Correct: [38.125 68.359375]% Cross Entropy Loss: [2.0765464 1.0089693] Correct: [35.390625 66.953125]% Cross Entropy Loss: [2.098847 0.94242144] Correct: [36.171875 67.578125]% Cross Entropy Loss: [2.1204734 1.0150812] Correct: [32.1875 65.3125]% Cross Entropy Loss: [1.9895303 0.9160978] Correct: [39.453125 68.515625]% Cross Entropy Loss: [1.9209778 0.8849082] Correct: [38.984375 69.0625 ]% Cross Entropy Loss: [2.0471623 0.98086417] Correct: [36.71875 67.34375]% Cross Entropy Loss: [2.0314407 0.961063 ] Correct: [36.640625 67.578125]% Cross Entropy Loss: [1.9328007 0.91085875] Correct: [40. 68.984375]% Cross Entropy Loss: [1.9666016 0.9720584] Correct: [36.40625 67.65625]% Cross Entropy Loss: [2.0907552 0.97460425] Correct: [34.84375 66.875 ]% Cross Entropy Loss: [1.9897633 0.90421486] Correct: [36.015625 68.90625 ]% Cross Entropy Loss: [2.0546653 0.9936339] Correct: [36.484375 66.640625]% Cross Entropy Loss: [2.094484 0.98361015] Correct: [36.953125 66.71875 ]% Cross Entropy Loss: [2.0127513 0.96544516] Correct: [36.640625 67.421875]% Cross Entropy Loss: [2.0442758 1.0683099] Correct: [34.453125 63.515625]% Cross Entropy Loss: [1.9474897 0.95986634] Correct: [37.890625 68.125 ]% Cross Entropy Loss: [2.0015848 0.93926084] Correct: [36.25 68.90625]% Cross Entropy Loss: [2.1317792 1.0527403] Correct: [35.9375 65.78125]% Cross Entropy Loss: [1.9316018 0.952427 ] Correct: [39.0625 68.125 ]% Cross Entropy Loss: [2.00638 0.96092737] Correct: [36.640625 67.421875]% Cross Entropy Loss: [1.8246237 0.8705657] Correct: [41.328125 69.6875 ]% Cross Entropy Loss: [1.8658307 0.8492869] Correct: [40.625 69.375]% Cross Entropy Loss: [2.0435443 0.95105237] Correct: [36.328125 68.125 ]% Cross Entropy Loss: [1.9486004 0.948599 ] Correct: [37.5 67.5]% Cross Entropy Loss: [2.0627909 0.9531562] Correct: [34.921875 66.796875]% Cross Entropy Loss: [1.9933479 0.9111034] Correct: [35.703125 68.984375]% Cross Entropy Loss: [2.025021 0.91847146] Correct: [37.34375 68.671875]% Cross Entropy Loss: [2.3270335 1.1761326] Correct: [31.5625 61.875 ]% Cross Entropy Loss: [1.9035137 0.9525846] Correct: [37.96875 67.890625]% Cross Entropy Loss: [2.020745 0.9784497] Correct: [36.484375 66.640625]% Cross Entropy Loss: [1.9699932 0.88107526] Correct: [39.453125 70.390625]% Cross Entropy Loss: [2.00168 0.959568] Correct: [35.9375 67.109375]% Cross Entropy Loss: [2.037626 0.97339535] Correct: [36.171875 67.5 ]% Cross Entropy Loss: [1.8798797 0.854812 ] Correct: [38.984375 70.859375]% Cross Entropy Loss: [2.1051388 0.9969858] Correct: [36.25 65.859375]% Cross Entropy Loss: [2.1753252 1.0333042] Correct: [35. 66.796875]% Cross Entropy Loss: [1.9631293 0.91759413] Correct: [37.1875 69.53125]% Cross Entropy Loss: [1.9367015 0.87649316] Correct: [39.21875 69.6875 ]% Cross Entropy Loss: [2.0667691 1.0418979] Correct: [36.640625 66.40625 ]% Cross Entropy Loss: [1.953772 1.0190605] Correct: [37.03125 65.234375]% Cross Entropy Loss: [1.886129 0.9237026] Correct: [40.625 66.5625]% Cross Entropy Loss: [2.0398211 0.97493684] Correct: [37.1875 65.546875]% Cross Entropy Loss: [1.8908637 0.8097534] Correct: [36.796875 71.5625 ]% Cross Entropy Loss: [1.8600327 0.7603122] Correct: [39.765625 72.96875 ]% Cross Entropy Loss: [2.103202 0.9985012] Correct: [35.625 66.640625]% Cross Entropy Loss: [2.159507 0.9664115] Correct: [34.0625 68.828125]% Cross Entropy Loss: [2.0943897 0.9971267] Correct: [34.921875 65.625 ]% Cross Entropy Loss: [2.063113 0.992488] Correct: [33.828125 66.328125]% Cross Entropy Loss: [1.9957758 0.96082366] Correct: [37.5 67.8125]% Cross Entropy Loss: [2.1952832 1.0325992] Correct: [33.828125 65. ]% Cross Entropy Loss: [2.049588 0.88670176] Correct: [32.96875 67.578125]% Cross Entropy Loss: [1.9218496 0.86230105] Correct: [38.671875 70.46875 ]% Cross Entropy Loss: [1.8784367 0.8190611] Correct: [39.765625 71.71875 ]% Cross Entropy Loss: [1.9980795 0.9758997] Correct: [37.34375 68.203125]% Cross Entropy Loss: [1.8787616 0.8918911] Correct: [38.984375 68.75 ]% Cross Entropy Loss: [1.9422083 0.8649384] Correct: [38.59375 70.546875]% Cross Entropy Loss: [1.8889859 0.85567033] Correct: [38.75 71.171875]% Cross Entropy Loss: [1.9246792 0.9097005] Correct: [37.5 69.765625]% Cross Entropy Loss: [2.0962558 0.97551453] Correct: [33.828125 67.734375]% Cross Entropy Loss: [2.0486484 0.94421977] Correct: [35.15625 69.21875]% Cross Entropy Loss: [1.9825137 0.89731896] Correct: [38.90625 69.0625 ]% Cross Entropy Loss: [2.0522528 0.918252 ] Correct: [35.9375 69.140625]% Cross Entropy Loss: [2.011265 0.87225235] Correct: [38.359375 69.84375 ]% Cross Entropy Loss: [1.8965483 0.8765501] Correct: [38.203125 69.453125]% Cross Entropy Loss: [1.8651396 0.8475057] Correct: [38.75 69.375]% Cross Entropy Loss: [1.863528 0.85645753] Correct: [39.21875 70.625 ]% Cross Entropy Loss: [1.8804566 0.90898335] Correct: [39.53125 71.015625]% Cross Entropy Loss: [2.0096335 0.9515972] Correct: [38.4375 68.671875]% Cross Entropy Loss: [1.7532084 0.79297614] Correct: [42.65625 73.828125]% Cross Entropy Loss: [1.9990971 0.9433101] Correct: [36.71875 67.734375]% Cross Entropy Loss: [1.9729093 0.97961235] Correct: [38.515625 66.328125]% Cross Entropy Loss: [2.123677 1.1054599] Correct: [35.46875 63.515625]% Cross Entropy Loss: [2.0971045 1.0025947] Correct: [35.390625 66.09375 ]% Cross Entropy Loss: [1.8413223 0.84008074] Correct: [37.734375 70. ]% Cross Entropy Loss: [1.844622 0.83326787] Correct: [39.375 71.171875]% Cross Entropy Loss: [1.9734838 0.9557816] Correct: [37.34375 68.59375]% Cross Entropy Loss: [2.0033917 0.8578272] Correct: [38.046875 71.5625 ]% Cross Entropy Loss: [2.0226612 0.961528 ] Correct: [36.09375 67.1875 ]% Cross Entropy Loss: [1.73587 0.7808001] Correct: [41.171875 71.71875 ]% Cross Entropy Loss: [1.9928377 0.9319927] Correct: [37.8125 68.90625]% Cross Entropy Loss: [2.001022 0.9561221] Correct: [37.5 68.203125]% Cross Entropy Loss: [2.0754712 0.9611322] Correct: [37.03125 64.921875]% Cross Entropy Loss: [1.9277353 0.82283497] Correct: [37.265625 71.171875]% Cross Entropy Loss: [2.051054 0.96825564] Correct: [35.625 68.203125]% Cross Entropy Loss: [1.939779 0.81244963] Correct: [39.296875 72.03125 ]% Cross Entropy Loss: [1.8817718 0.8325998] Correct: [38.4375 72.34375]% Cross Entropy Loss: [1.984957 0.8346429] Correct: [37.109375 70.625 ]% Cross Entropy Loss: [1.8502548 0.77026904] Correct: [40.390625 74.6875 ]% Cross Entropy Loss: [1.9609667 0.8666066] Correct: [37.8125 70.859375]% Cross Entropy Loss: [1.9260147 0.79586405] Correct: [37.8125 72.65625]% Cross Entropy Loss: [1.8936918 0.90379256] Correct: [37.890625 70.78125 ]% Cross Entropy Loss: [1.8732399 0.8267051] Correct: [40.9375 72.5 ]% Cross Entropy Loss: [1.989661 0.9090274] Correct: [37.5 69.609375]% Cross Entropy Loss: [2.179319 1.0678756] Correct: [34.53125 65.9375 ]% Cross Entropy Loss: [1.9578682 0.89761484] Correct: [37.03125 68.359375]% Cross Entropy Loss: [1.9624512 0.90796673] Correct: [37.421875 71.09375 ]% Cross Entropy Loss: [1.9122375 0.85218143] Correct: [38.125 69.84375]% Cross Entropy Loss: [2.0853138 1.0030634] Correct: [36.953125 68.046875]% Cross Entropy Loss: [1.9190973 0.8771528] Correct: [37.1875 71.328125]% Cross Entropy Loss: [1.9544914 0.86003494] Correct: [39.53125 71.015625]% Cross Entropy Loss: [1.8948532 0.8068183] Correct: [38.59375 72.96875]% Cross Entropy Loss: [1.9353803 0.84142685] Correct: [38.4375 72.1875]% Cross Entropy Loss: [1.9477952 0.83895195] Correct: [38.125 70.78125]% Cross Entropy Loss: [1.9227444 0.8735463] Correct: [38.515625 70.390625]% Cross Entropy Loss: [1.9804776 0.8924748] Correct: [37.578125 67.1875 ]% Cross Entropy Loss: [1.9269329 0.819456 ] Correct: [38.90625 72.421875]% Cross Entropy Loss: [1.9962101 0.8835033] Correct: [37.578125 70.3125 ]% Cross Entropy Loss: [1.8087041 0.785474 ] Correct: [38.984375 71.5625 ]% Cross Entropy Loss: [2.03966 0.9373995] Correct: [35. 67.5]% Cross Entropy Loss: [2.0304132 0.90852773] Correct: [36.71875 69.296875]% Cross Entropy Loss: [1.905365 0.84209096] Correct: [41.25 72.265625]% Cross Entropy Loss: [1.9505103 0.8191303] Correct: [37.65625 73.125 ]% Cross Entropy Loss: [2.0503025 0.87213975] Correct: [35.234375 69.0625 ]% Cross Entropy Loss: [1.9317973 0.84376764] Correct: [39.765625 71.640625]% Cross Entropy Loss: [2.014137 0.87423134] Correct: [36.5625 69.375 ]% Cross Entropy Loss: [1.9005562 0.84304696] Correct: [39.453125 70.46875 ]% Cross Entropy Loss: [2.0660887 0.99748594] Correct: [36.015625 65.15625 ]% Cross Entropy Loss: [2.1215038 0.98842144] Correct: [33.75 66.796875]% Cross Entropy Loss: [2.0807695 0.9069824] Correct: [36.171875 69.375 ]% Cross Entropy Loss: [1.9273688 0.86544514] Correct: [38.515625 70.859375]% Cross Entropy Loss: [1.9238075 0.85757244] Correct: [37.578125 70.625 ]% Cross Entropy Loss: [2.0320601 0.89199555] Correct: [35.9375 69.453125]% Cross Entropy Loss: [2.05024 0.98934346] Correct: [36.640625 67.8125 ]% Cross Entropy Loss: [1.7614399 0.75944966] Correct: [42.734375 73.515625]% Cross Entropy Loss: [1.925534 0.85670435] Correct: [38.828125 71.875 ]% Cross Entropy Loss: [1.9007986 0.8332615] Correct: [40.46875 72.421875]% Cross Entropy Loss: [1.9186542 0.83438635] Correct: [39.921875 70.546875]% Cross Entropy Loss: [1.8746021 0.8211813] Correct: [41.171875 72.265625]% Cross Entropy Loss: [1.7602314 0.79013455] Correct: [42.96875 73.4375 ]% Cross Entropy Loss: [1.8850124 0.8376728] Correct: [39.21875 71.71875]% Cross Entropy Loss: [1.811408 0.7245079] Correct: [40.78125 75.46875]% Cross Entropy Loss: [1.9930766 0.90081185] Correct: [35.078125 70.703125]% Cross Entropy Loss: [2.0154529 0.901935 ] Correct: [35.625 69.0625]% Cross Entropy Loss: [2.180222 1.0402296] Correct: [33.90625 63.984375]% Cross Entropy Loss: [2.042675 0.92094195] Correct: [35.78125 69.296875]% Cross Entropy Loss: [1.9534737 0.851793 ] Correct: [38.046875 71.171875]% Cross Entropy Loss: [1.9514538 0.85383034] Correct: [38.828125 71.09375 ]% Cross Entropy Loss: [1.8584185 0.7944455] Correct: [38.125 73.203125]% Cross Entropy Loss: [1.9999424 0.9131092] Correct: [37.421875 69.296875]% Cross Entropy Loss: [1.9190003 0.847613 ] Correct: [38.359375 72.5 ]% Cross Entropy Loss: [2.0072734 0.92615974] Correct: [37.1875 70.3125]% Cross Entropy Loss: [1.9437368 0.8993247] Correct: [36.953125 70.3125 ]% Cross Entropy Loss: [1.9171613 0.81850755] Correct: [36.5625 72.109375]% Cross Entropy Loss: [1.9808207 0.8681512] Correct: [36.484375 70.625 ]% Cross Entropy Loss: [1.8110718 0.78174305] Correct: [41.5625 72.734375]% Cross Entropy Loss: [2.125862 1.000222] Correct: [34.53125 65.390625]% Cross Entropy Loss: [2.0461168 0.8822401] Correct: [35.625 69.765625]% Cross Entropy Loss: [2.0368335 0.9592369] Correct: [36.171875 68.28125 ]% Cross Entropy Loss: [1.9582134 0.91381055] Correct: [37.890625 70.15625 ]% Cross Entropy Loss: [1.8364515 0.8389826] Correct: [40.078125 70.703125]% Cross Entropy Loss: [1.8883171 0.7942802] Correct: [36.875 71.71875]% Cross Entropy Loss: [1.9808524 0.830274 ] Correct: [37.578125 71.25 ]% Cross Entropy Loss: [2.0649805 0.9389828] Correct: [34.53125 69.6875 ]% Cross Entropy Loss: [1.9704504 0.8745142] Correct: [36.5625 70.859375]% Cross Entropy Loss: [1.9719826 0.90744084] Correct: [36.484375 68.59375 ]% Cross Entropy Loss: [1.7403939 0.71775883] Correct: [42.578125 76.640625]% Cross Entropy Loss: [2.0806084 0.96995324] Correct: [34.6875 68.515625]% Cross Entropy Loss: [1.8309376 0.73546416] Correct: [38.671875 74.140625]% Cross Entropy Loss: [1.7746108 0.7507173] Correct: [43.046875 74.453125]% Cross Entropy Loss: [1.8725427 0.7845746] Correct: [39.6875 73.59375]% Cross Entropy Loss: [2.0077446 0.9015192] Correct: [36.953125 69.296875]% Cross Entropy Loss: [1.8845736 0.8225733] Correct: [38.125 72.890625]% Cross Entropy Loss: [1.8909432 0.8260232] Correct: [39.140625 71.015625]% Cross Entropy Loss: [1.9915955 0.9004551] Correct: [37.421875 69.765625]% Cross Entropy Loss: [2.0216527 0.9416069] Correct: [36.875 67.8125]% Cross Entropy Loss: [2.0008621 0.9356631] Correct: [35.390625 66.015625]% Cross Entropy Loss: [1.9996936 0.87418413] Correct: [37.265625 70.546875]% Cross Entropy Loss: [2.0175574 0.8969595] Correct: [35.15625 69.296875]% Cross Entropy Loss: [2.1736145 0.9772995] Correct: [32.1875 67.578125]% Cross Entropy Loss: [1.8958956 0.77489245] Correct: [38.4375 73.671875]% Cross Entropy Loss: [1.9124041 0.8112958] Correct: [38.59375 72.890625]% Cross Entropy Loss: [1.7836368 0.81923103] Correct: [40.078125 73.59375 ]% Cross Entropy Loss: [1.8672546 0.7744461] Correct: [41.5625 74.140625]% Cross Entropy Loss: [1.8219055 0.7922637] Correct: [38.90625 73.671875]% Cross Entropy Loss: [2.0713143 0.8693514] Correct: [35.546875 70.625 ]% Cross Entropy Loss: [1.8485124 0.85592985] Correct: [38.59375 70.9375 ]% Cross Entropy Loss: [1.855629 0.8751441] Correct: [39.765625 71.171875]% Cross Entropy Loss: [1.9413006 0.8784059] Correct: [38.203125 70.46875 ]% Cross Entropy Loss: [2.001966 0.9464778] Correct: [37.890625 69.6875 ]% Cross Entropy Loss: [1.8575532 0.7590194] Correct: [39.921875 75.15625 ]% Cross Entropy Loss: [1.8918136 0.8250167] Correct: [39.921875 73.046875]% Cross Entropy Loss: [1.9186437 0.85178596] Correct: [38.671875 72.8125 ]% Cross Entropy Loss: [2.0408514 0.9232028] Correct: [36.484375 70.078125]% Cross Entropy Loss: [1.896796 0.84057844] Correct: [38.75 71.25]% Cross Entropy Loss: [1.9959234 0.8758749] Correct: [36.953125 70.15625 ]% Cross Entropy Loss: [2.0072932 0.88783616] Correct: [37.578125 69.765625]% Cross Entropy Loss: [1.74315 0.7753718] Correct: [41.09375 73.75 ]% Cross Entropy Loss: [1.8673702 0.7675727] Correct: [39.53125 73.984375]% Cross Entropy Loss: [2.0016038 0.89843196] Correct: [38.125 69.921875]% Cross Entropy Loss: [2.001402 0.8990204] Correct: [36.71875 69.6875 ]% Cross Entropy Loss: [1.7711271 0.7267686] Correct: [42.03125 75.546875]% Cross Entropy Loss: [1.9994249 0.8564911] Correct: [38.125 70.9375]% Cross Entropy Loss: [1.9475883 0.8168581] Correct: [39.6875 73.046875]% Cross Entropy Loss: [1.8392327 0.7327321] Correct: [39.84375 76.328125]% Cross Entropy Loss: [1.8312792 0.74983966] Correct: [41.796875 73.75 ]% Cross Entropy Loss: [1.7291515 0.7506291] Correct: [42.34375 74.0625 ]% Cross Entropy Loss: [2.0258403 0.87854844] Correct: [35.234375 70.859375]% Cross Entropy Loss: [1.9590372 0.83854675] Correct: [36.171875 72.578125]% Cross Entropy Loss: [1.8812119 0.8078213] Correct: [38.828125 73.125 ]% Cross Entropy Loss: [1.7979572 0.6917948] Correct: [39.84375 76.71875]% Cross Entropy Loss: [1.9815601 0.83466244] Correct: [39.21875 71.25 ]% Cross Entropy Loss: [1.8359292 0.8135235] Correct: [39.375 72.421875]% Cross Entropy Loss: [1.8439033 0.8367712] Correct: [40.234375 71.171875]% Cross Entropy Loss: [1.8491738 0.772687 ] Correct: [40.9375 74.609375]% Cross Entropy Loss: [1.7755102 0.7650427] Correct: [42.65625 74.6875 ]% Cross Entropy Loss: [1.7570293 0.72656125] Correct: [43.125 76.5625]% Cross Entropy Loss: [1.9024506 0.74392164] Correct: [41.875 74.296875]% Cross Entropy Loss: [1.7916753 0.71454906] Correct: [41.40625 76.875 ]% Cross Entropy Loss: [1.7586281 0.7206019] Correct: [39.84375 75.625 ]% Cross Entropy Loss: [1.9066198 0.87098444] Correct: [38.671875 71.640625]% Cross Entropy Loss: [1.8574002 0.82841396] Correct: [41.015625 72.421875]% Cross Entropy Loss: [1.9432628 0.82397985] Correct: [37.421875 72.5 ]% Cross Entropy Loss: [1.845761 0.8483675] Correct: [40.234375 71.484375]% Cross Entropy Loss: [1.7709032 0.7465607] Correct: [42.03125 73.4375 ]% Cross Entropy Loss: [2.010252 0.9226085] Correct: [37.890625 68.984375]% Cross Entropy Loss: [1.8785301 0.7842553] Correct: [38.671875 72.65625 ]% Cross Entropy Loss: [1.8122343 0.8174494] Correct: [41.25 72.03125]% Cross Entropy Loss: [1.8043875 0.73387396] Correct: [39.375 75.234375]% Cross Entropy Loss: [1.9359577 0.8675893] Correct: [39.140625 70.78125 ]% Cross Entropy Loss: [2.0243871 0.86184883] Correct: [35.859375 71.171875]% Cross Entropy Loss: [1.7832315 0.75763065] Correct: [40.859375 74.609375]% Cross Entropy Loss: [2.0323367 0.9217459] Correct: [35.625 67.96875]% Cross Entropy Loss: [1.8078743 0.7337283] Correct: [40.234375 75.234375]% Cross Entropy Loss: [2.095902 0.91733587] Correct: [37.5 68.90625]% Cross Entropy Loss: [1.8610016 0.7914856] Correct: [38.28125 73.90625]% Cross Entropy Loss: [1.9666817 0.76745856] Correct: [38.828125 75.078125]% Cross Entropy Loss: [1.8362296 0.7332736] Correct: [39.21875 73.90625]% Cross Entropy Loss: [1.9611698 0.8414934] Correct: [37.734375 72.421875]% Cross Entropy Loss: [1.9443098 0.84748536] Correct: [36.5625 70.546875]% Cross Entropy Loss: [1.7100394 0.73157245] Correct: [42.5 75.390625]% Cross Entropy Loss: [1.7344234 0.686546 ] Correct: [41.40625 75.390625]% Cross Entropy Loss: [2.2196267 0.9661806] Correct: [33.203125 67.8125 ]% Cross Entropy Loss: [1.8975483 0.8913821] Correct: [39.21875 68.515625]% Cross Entropy Loss: [2.0402424 0.8897041] Correct: [35.234375 70.703125]% Cross Entropy Loss: [1.9627292 0.76570493] Correct: [37.96875 74.21875]% Cross Entropy Loss: [1.9946296 0.85147095] Correct: [37.8125 71.640625]% Cross Entropy Loss: [1.9930117 0.8758448] Correct: [36.40625 69.84375]% Cross Entropy Loss: [1.8006637 0.7793001] Correct: [40.78125 73.828125]% Cross Entropy Loss: [1.8989598 0.7816679] Correct: [38.90625 75.3125 ]% Cross Entropy Loss: [1.9885483 0.8509863] Correct: [35.9375 71.953125]% Cross Entropy Loss: [1.9961727 0.81813097] Correct: [35. 72.65625]% Cross Entropy Loss: [1.9514248 0.8419062] Correct: [35.625 72.5 ]% Cross Entropy Loss: [1.9268863 0.8127308] Correct: [38.046875 72.5 ]% Cross Entropy Loss: [1.8251534 0.749963 ] Correct: [38.59375 74.296875]% Cross Entropy Loss: [1.847708 0.80535996] Correct: [38.75 73.984375]% Cross Entropy Loss: [1.77269 0.8101282] Correct: [42.109375 73.4375 ]% Cross Entropy Loss: [1.851477 0.7737516] Correct: [40.3125 73.359375]% Cross Entropy Loss: [1.8459237 0.7547766] Correct: [39.6875 74.921875]% Cross Entropy Loss: [1.96306 0.77251136] Correct: [36.40625 74.0625 ]% Cross Entropy Loss: [1.847846 0.7621843] Correct: [40.3125 74.375 ]% Cross Entropy Loss: [2.021227 0.84209365] Correct: [36.328125 72.1875 ]% Cross Entropy Loss: [1.8221203 0.75928843] Correct: [39.53125 73.75 ]% Cross Entropy Loss: [1.941445 0.7407073] Correct: [37.5 76.171875]% Cross Entropy Loss: [1.9379956 0.7936501] Correct: [37.1875 72.890625]% Cross Entropy Loss: [1.7764927 0.7348779] Correct: [43.046875 74.765625]% Cross Entropy Loss: [1.9029028 0.7937895] Correct: [38.90625 72.5 ]% Cross Entropy Loss: [1.7761481 0.7383987] Correct: [39.84375 74.453125]% Cross Entropy Loss: [1.9013875 0.79574096] Correct: [39.21875 73.59375]% Cross Entropy Loss: [1.8257484 0.77694726] Correct: [39.140625 73.75 ]% Cross Entropy Loss: [1.8944756 0.776986 ] Correct: [40.78125 73.359375]% Cross Entropy Loss: [1.8227527 0.80487216] Correct: [37.890625 72.421875]% Cross Entropy Loss: [2.0018544 0.90893173] Correct: [35.390625 68.125 ]% Cross Entropy Loss: [1.9835513 0.8218819] Correct: [37.890625 71.09375 ]% Cross Entropy Loss: [1.887186 0.898434] Correct: [40.15625 69.53125]% Cross Entropy Loss: [1.8470852 0.83126515] Correct: [39.921875 72.8125 ]% Cross Entropy Loss: [1.7540865 0.8286497] Correct: [41.953125 71.71875 ]% Cross Entropy Loss: [1.8571377 0.7411334] Correct: [38.515625 75.3125 ]% Cross Entropy Loss: [1.9381046 0.76549727] Correct: [38.59375 72.8125 ]% Cross Entropy Loss: [1.7097151 0.69064677] Correct: [43.671875 76.484375]% Cross Entropy Loss: [2.0125422 0.8248552] Correct: [35.390625 72.34375 ]% Cross Entropy Loss: [1.8507038 0.7185246] Correct: [38.75 75.78125]% Cross Entropy Loss: [1.8475326 0.76161605] Correct: [40.234375 73.28125 ]% Cross Entropy Loss: [1.7858212 0.80732393] Correct: [40.546875 72.109375]% Cross Entropy Loss: [1.9122384 0.8764744] Correct: [39.0625 71.40625]% Cross Entropy Loss: [1.8023087 0.7331826] Correct: [39.140625 75.390625]% Cross Entropy Loss: [1.9203488 0.8596018] Correct: [37.8125 71.171875]% Cross Entropy Loss: [1.7938801 0.7361818] Correct: [40.78125 75.703125]% Cross Entropy Loss: [1.8634443 0.72198576] Correct: [37.5 75.3125]% Cross Entropy Loss: [1.951506 0.77085984] Correct: [38.203125 72.96875 ]% Cross Entropy Loss: [1.9493883 0.7860302] Correct: [38.90625 73.59375]% Cross Entropy Loss: [1.9975573 0.8228611] Correct: [35.859375 71.25 ]% Cross Entropy Loss: [1.8333766 0.7738951] Correct: [41.328125 73.90625 ]% Cross Entropy Loss: [1.9985955 0.8937216] Correct: [35.703125 68.90625 ]% Cross Entropy Loss: [1.6448488 0.71342045] Correct: [42.421875 75.859375]% Cross Entropy Loss: [1.9145508 0.79811525] Correct: [37.109375 73.59375 ]% Cross Entropy Loss: [1.8653772 0.78147805] Correct: [40. 73.90625]% Cross Entropy Loss: [1.8586502 0.82692623] Correct: [38.359375 73.359375]% Cross Entropy Loss: [2.0272176 0.90049267] Correct: [34.921875 69.296875]% Cross Entropy Loss: [1.8368189 0.7983962] Correct: [42.1875 71.71875]%
0%| | 0/36 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 46.95% From layer 1: Accuracy: 68.86%
0%| | 0/128 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 45.13% From layer 1: Accuracy: 78.47%
accs = {'train': train_acc, 'test': test_acc}
with open('final_results/' + model_name[:-3] + '_accuracies.pkl', 'wb') as f:
pickle.dump(accs, f)
if not args.augment or True:
# if snn_samples already exist, don't recompute
accs = {}
try:
snn_samples
except:
snn_samples, targets = get_samples(SNN, train_loader, args.n_hidden, device)
for cat in [False, True]:
test_accs = []
train_accs = []
for i in range(10):
with torch.no_grad():
out_projs, acc, losses_out = train_out_proj_fast(SNN, args, 30, 60, snn_samples, train_loader.y, cat=cat, lr=3e-4, weight_decay=0)
print('Mean abs weights', out_projs[-1].out_proj.weight.abs().mean())
test_accs.append(get_accuracy(SNN, out_projs, test_loader, device, cat=cat)[0])
train_accs.append(get_accuracy(SNN, out_projs, train_loader, device, cat=cat)[0])
test_accs = torch.stack([torch.tensor(ta) for ta in test_accs])
train_accs = torch.stack([torch.tensor(ta) for ta in train_accs])
print(f'Fast Classifier Mean Test Accuracy: {100*torch.mean(test_accs, dim=0)}, Std: {100*torch.std(test_accs, dim=0)}')
print(f'Fast Classifier Mean Train Accuracy: {100*torch.mean(train_accs, dim=0)}, Std: {100*torch.std(train_accs, dim=0)}')
if cat:
accs['train_cat'] = train_accs
accs['test_cat'] = test_accs
else:
accs['train'] = train_accs
accs['test'] = test_accs
with open('final_results/' + model_name[:-3] + '_accuracies.pkl', 'wb') as f:
pickle.dump(accs, f)
0%| | 0/127 [00:00<?, ?it/s]
0%| | 0/30 [00:00<?, ?it/s]
Cross Entropy Loss: [4.242321 3.597268] Correct: [16.01275135 25.64982835]% Cross Entropy Loss: [2.6065571 1.7033745] Correct: [28.28592447 47.49877391]% Cross Entropy Loss: [2.308533 1.3958906] Correct: [33.14124571 56.33889161]% Cross Entropy Loss: [2.035802 1.1075165] Correct: [38.46248161 63.2540461 ]% Cross Entropy Loss: [1.8709322 0.9424524] Correct: [41.67484061 68.23197646]% Cross Entropy Loss: [1.740124 0.8482229] Correct: [44.62972045 71.97155468]% Cross Entropy Loss: [1.6877215 0.76048875] Correct: [45.81902894 74.86512997]% Cross Entropy Loss: [1.591468 0.6946195] Correct: [48.62677783 76.8023541 ]% Cross Entropy Loss: [1.5229774 0.6221868] Correct: [50.51495831 79.07062285]% Cross Entropy Loss: [1.458724 0.5892946] Correct: [52.0107896 80.8239333]% Cross Entropy Loss: [1.3771989 0.49151567] Correct: [54.04610103 83.96272683]% Cross Entropy Loss: [1.3287163 0.45680276] Correct: [55.37027955 85.26238352]% Cross Entropy Loss: [1.2989422 0.48010737] Correct: [56.3266307 84.47768514]% Cross Entropy Loss: [1.2240931 0.4037957] Correct: [58.3374203 87.13830309]% Cross Entropy Loss: [1.1788702 0.3904544] Correct: [59.96812163 87.3467386 ]% Cross Entropy Loss: [1.1600385 0.41012552] Correct: [60.07846984 86.01029917]% Cross Entropy Loss: [1.1524144 0.3574405] Correct: [60.92447278 87.98430603]% Cross Entropy Loss: [1.0817355 0.3115347] Correct: [62.39578225 90.15448749]% Cross Entropy Loss: [1.0559404 0.31925252] Correct: [63.36439431 89.97057381]% Cross Entropy Loss: [1.0346327 0.31040132] Correct: [63.73222168 89.81118195]% Cross Entropy Loss: [1.0102172 0.27367902] Correct: [64.92153016 91.29475233]% Cross Entropy Loss: [0.9743725 0.2954081] Correct: [66.35605689 90.8656204 ]% Cross Entropy Loss: [0.98448575 0.29440653] Correct: [66.20892594 90.52231486]% Cross Entropy Loss: [0.94053906 0.25010514] Correct: [67.10397254 92.31240804]% Cross Entropy Loss: [0.94230896 0.23350498] Correct: [67.54536538 92.93771457]% Cross Entropy Loss: [0.92381734 0.22419865] Correct: [68.41589014 93.35458558]% Cross Entropy Loss: [0.87555194 0.22176147] Correct: [69.75232957 93.21971555]% Cross Entropy Loss: [0.85328805 0.19395205] Correct: [70.32859245 94.21284944]% Cross Entropy Loss: [0.85153246 0.18605126] Correct: [70.97842079 94.33545856]% Cross Entropy Loss: [0.84349555 0.17225222] Correct: [71.00294262 95.07111329]% Mean abs weights tensor(0.0282, grad_fn=<MeanBackward0>)
0%| | 0/36 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 42.49% From layer 1: Accuracy: 59.28%
0%| | 0/128 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 68.37% From layer 1: Accuracy: 94.26%
0%| | 0/30 [00:00<?, ?it/s]
Cross Entropy Loss: [4.073899 3.084285] Correct: [17.04266797 27.88131437]% Cross Entropy Loss: [2.4571614 1.6696565] Correct: [29.83079941 48.1976459 ]% Cross Entropy Loss: [2.158677 1.3225952] Correct: [35.96125552 57.30750368]% Cross Entropy Loss: [1.9980195 1.1241448] Correct: [40.17900932 63.14369789]% Cross Entropy Loss: [1.8545433 0.9479055] Correct: [43.0725846 68.86954389]% Cross Entropy Loss: [1.7089391 0.810201 ] Correct: [45.61059343 72.63364394]% Cross Entropy Loss: [1.6610197 0.74785984] Correct: [47.51103482 74.77930358]% Cross Entropy Loss: [1.5396955 0.6747855] Correct: [50.20843551 76.71652771]% Cross Entropy Loss: [1.4511102 0.57624143] Correct: [52.55026974 80.48062776]% Cross Entropy Loss: [1.4498479 0.54617566] Correct: [52.57479156 82.17263364]% Cross Entropy Loss: [1.3869054 0.5143402] Correct: [54.3158411 83.6316822]% Cross Entropy Loss: [1.3152362 0.4659862] Correct: [56.71897989 84.22020598]% Cross Entropy Loss: [1.2791835 0.4460537] Correct: [56.66993624 84.93133889]% Cross Entropy Loss: [1.188095 0.4030684] Correct: [59.35507602 86.7459539 ]% Cross Entropy Loss: [1.210667 0.3967458] Correct: [59.15890142 87.07699853]% Cross Entropy Loss: [1.1536107 0.39789623] Correct: [60.7405591 86.96665032]% Cross Entropy Loss: [1.1533293 0.33190638] Correct: [60.63021089 89.60274644]% Cross Entropy Loss: [1.0722674 0.33376732] Correct: [63.67091712 89.36978911]% Cross Entropy Loss: [1.0457838 0.3104883] Correct: [63.92839627 90.20353114]% Cross Entropy Loss: [1.044988 0.2806283] Correct: [65.10544385 91.07405591]% Cross Entropy Loss: [0.99554724 0.24690443] Correct: [66.061795 92.42275625]% Cross Entropy Loss: [0.97746533 0.26394287] Correct: [66.52770966 91.49092692]% Cross Entropy Loss: [0.95890874 0.24020644] Correct: [67.25110348 92.6679745 ]% Cross Entropy Loss: [0.95711505 0.25717342] Correct: [67.12849436 91.85875429]% Cross Entropy Loss: [0.9108462 0.2540686] Correct: [68.90632663 91.95684159]% Cross Entropy Loss: [0.9289546 0.21630014] Correct: [67.82736636 93.29328102]% Cross Entropy Loss: [0.90583116 0.20905387] Correct: [68.74693477 93.41589014]% Cross Entropy Loss: [0.85201806 0.17822443] Correct: [70.36537518 94.85041687]% Cross Entropy Loss: [0.8225846 0.18970598] Correct: [71.83668465 94.32319765]% Cross Entropy Loss: [0.8006406 0.18231758] Correct: [71.92251103 94.37224129]% Mean abs weights tensor(0.0284, grad_fn=<MeanBackward0>)
0%| | 0/36 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 45.58% From layer 1: Accuracy: 60.20%
0%| | 0/128 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 72.36% From layer 1: Accuracy: 94.04%
0%| | 0/30 [00:00<?, ?it/s]
Cross Entropy Loss: [4.817585 3.1494026] Correct: [16.33153507 26.90044139]% Cross Entropy Loss: [2.5496001 1.720337 ] Correct: [29.10740559 47.78077489]% Cross Entropy Loss: [2.1929011 1.2976797] Correct: [35.18881805 58.5335949 ]% Cross Entropy Loss: [1.9794525 1.1509584] Correct: [39.68857283 63.37665522]% Cross Entropy Loss: [1.8100059 0.9004014] Correct: [42.91319274 70.14467876]% Cross Entropy Loss: [1.7021078 0.8466746] Correct: [45.29180971 71.66503188]% Cross Entropy Loss: [1.5714142 0.7334806] Correct: [49.25208436 75.82148112]% Cross Entropy Loss: [1.5303166 0.65792185] Correct: [50. 77.74644434]% Cross Entropy Loss: [1.4232478 0.6005354] Correct: [52.48896518 80.08827857]% Cross Entropy Loss: [1.4100878 0.557266 ] Correct: [52.74644434 81.17949975]% Cross Entropy Loss: [1.3389598 0.52500254] Correct: [55.37027955 82.49141736]% Cross Entropy Loss: [1.2852685 0.4657681] Correct: [56.65767533 85.0416871 ]% Cross Entropy Loss: [1.2536222 0.43154344] Correct: [57.55272192 85.36047082]% Cross Entropy Loss: [1.1850439 0.39446685] Correct: [60.07846984 87.10152035]% Cross Entropy Loss: [1.1486057 0.39163616] Correct: [61.36586562 87.27317312]% Cross Entropy Loss: [1.138888 0.33588448] Correct: [61.78273664 89.1613536 ]% Cross Entropy Loss: [1.1087989 0.33060727] Correct: [61.96665032 89.22265817]% Cross Entropy Loss: [1.0505157 0.34494877] Correct: [63.91613536 88.60961256]% Cross Entropy Loss: [1.0593307 0.2977021] Correct: [63.87935262 90.52231486]% Cross Entropy Loss: [1.0332059 0.29888126] Correct: [64.67631192 90.15448749]% Cross Entropy Loss: [0.9646623 0.26796493] Correct: [66.50318784 91.25796959]% Cross Entropy Loss: [1.000825 0.27082965] Correct: [65.92692496 91.71162334]% Cross Entropy Loss: [0.9209945 0.22360133] Correct: [69.00441393 93.33006376]% Cross Entropy Loss: [0.9411144 0.22391175] Correct: [67.4717999 93.08484551]% Cross Entropy Loss: [0.8951251 0.2265266] Correct: [69.42128494 93.28102011]% Cross Entropy Loss: [0.8751516 0.18989967] Correct: [69.44580677 94.42128494]% Cross Entropy Loss: [0.86485827 0.20826614] Correct: [69.75232957 93.58754291]% Cross Entropy Loss: [0.8685733 0.20807831] Correct: [70.26728789 93.73467386]% Cross Entropy Loss: [0.82083774 0.17242521] Correct: [71.65277097 95.07111329]% Cross Entropy Loss: [0.79769623 0.24366604] Correct: [72.42520844 92.6679745 ]% Mean abs weights tensor(0.0282, grad_fn=<MeanBackward0>)
0%| | 0/36 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 45.19% From layer 1: Accuracy: 62.46%
0%| | 0/128 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 73.31% From layer 1: Accuracy: 94.62%
0%| | 0/30 [00:00<?, ?it/s]
Cross Entropy Loss: [4.327544 3.121484] Correct: [16.18440412 27.15792055]% Cross Entropy Loss: [2.4879038 1.7302457] Correct: [29.95340853 47.17999019]% Cross Entropy Loss: [2.1704276 1.323708 ] Correct: [35.82638548 57.14811182]% Cross Entropy Loss: [1.9856046 1.0945578] Correct: [39.72535557 63.89161354]% Cross Entropy Loss: [1.8662959 0.9745579] Correct: [43.39136832 68.13388916]% Cross Entropy Loss: [1.7445215 0.87123567] Correct: [45.23050515 71.72633644]% Cross Entropy Loss: [1.6118568 0.696673 ] Correct: [48.13634134 76.37322217]% Cross Entropy Loss: [1.5618907 0.695606 ] Correct: [49.74252084 76.55713585]% Cross Entropy Loss: [1.4951458 0.58966815] Correct: [52.12113781 80.89749877]% Cross Entropy Loss: [1.4569771 0.5676034] Correct: [52.63609613 80.95880333]% Cross Entropy Loss: [1.3763602 0.51568174] Correct: [55.1250613 82.99411476]% Cross Entropy Loss: [1.3067043 0.47274423] Correct: [56.60863168 84.67385974]% Cross Entropy Loss: [1.2475032 0.43454126] Correct: [58.32515939 85.69151545]% Cross Entropy Loss: [1.2692385 0.40325662] Correct: [58.54585581 86.66012751]% Cross Entropy Loss: [1.1839606 0.40141073] Correct: [60.2378617 86.50073565]% Cross Entropy Loss: [1.1717148 0.38092324] Correct: [61.19421285 87.53065228]% Cross Entropy Loss: [1.1394297 0.33831626] Correct: [61.47621383 89.36978911]% Cross Entropy Loss: [1.1169313 0.3335592] Correct: [61.88082393 89.08778813]% Cross Entropy Loss: [1.0985036 0.35411304] Correct: [62.849436 88.48700343]% Cross Entropy Loss: [1.000474 0.27461743] Correct: [65.62040216 91.29475233]% Cross Entropy Loss: [1.0095401 0.2693873] Correct: [65.63266307 91.3683178 ]% Cross Entropy Loss: [0.97859585 0.2549964 ] Correct: [66.41736145 92.1652771 ]% Cross Entropy Loss: [0.9864843 0.26967803] Correct: [66.51544875 91.29475233]% Cross Entropy Loss: [0.9471579 0.22369798] Correct: [67.32466896 93.14615007]% Cross Entropy Loss: [0.929046 0.21375218] Correct: [67.87641 93.61206474]% Cross Entropy Loss: [0.9104445 0.20008993] Correct: [68.79597842 94.28641491]% Cross Entropy Loss: [0.8795324 0.19315547] Correct: [70.05885238 94.54389407]% Cross Entropy Loss: [0.8492627 0.19857624] Correct: [70.69641981 94.04119667]% Cross Entropy Loss: [0.8293602 0.1812293] Correct: [71.56694458 94.69102501]% Cross Entropy Loss: [0.8178792 0.17953587] Correct: [71.787641 94.72780775]% Mean abs weights tensor(0.0281, grad_fn=<MeanBackward0>)
0%| | 0/36 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 45.32% From layer 1: Accuracy: 63.91%
0%| | 0/128 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 72.17% From layer 1: Accuracy: 97.52%
0%| | 0/30 [00:00<?, ?it/s]
Cross Entropy Loss: [4.070601 3.1317592] Correct: [16.44188328 24.79156449]% Cross Entropy Loss: [2.4527628 1.6880032] Correct: [30.23540951 47.81755763]% Cross Entropy Loss: [2.1775837 1.317041 ] Correct: [35.77734183 56.62089259]% Cross Entropy Loss: [1.9400244 1.1106669] Correct: [39.94605199 63.53604708]% Cross Entropy Loss: [1.8046176 0.93704015] Correct: [43.30554193 68.89406572]% Cross Entropy Loss: [1.6927974 0.83194435] Correct: [45.91711623 72.20451202]% Cross Entropy Loss: [1.606057 0.7346853] Correct: [49.05590976 74.97547818]% Cross Entropy Loss: [1.5419457 0.63257295] Correct: [50.80922021 78.75183914]% Cross Entropy Loss: [1.4753547 0.6120853] Correct: [52.62383521 79.80627759]% Cross Entropy Loss: [1.4268469 0.5294239] Correct: [53.20009809 82.12359 ]% Cross Entropy Loss: [1.3741575 0.5099843] Correct: [55.52967141 82.82246199]% Cross Entropy Loss: [1.3249209 0.49259162] Correct: [56.58410986 83.69298676]% Cross Entropy Loss: [1.2742426 0.4784017] Correct: [57.79794017 83.85237862]% Cross Entropy Loss: [1.2393807 0.42719865] Correct: [59.31829328 85.7405591 ]% Cross Entropy Loss: [1.1515131 0.35231334] Correct: [61.25551741 88.57282982]% Cross Entropy Loss: [1.1796793 0.34744847] Correct: [60.6424718 88.94065718]% Cross Entropy Loss: [1.0937381 0.3324708] Correct: [63.43795978 89.28396273]% Cross Entropy Loss: [1.0860159 0.33217537] Correct: [63.54830799 88.78126533]% Cross Entropy Loss: [1.0553398 0.30146846] Correct: [64.43109367 90.35066209]% Cross Entropy Loss: [1.0420817 0.2994347] Correct: [64.73761648 90.31387935]% Cross Entropy Loss: [1.0087656 0.27375016] Correct: [66.35605689 91.76066699]% Cross Entropy Loss: [0.98227817 0.24844377] Correct: [66.82197155 92.21432075]% Cross Entropy Loss: [0.95235205 0.23944178] Correct: [67.61893085 92.6679745 ]% Cross Entropy Loss: [0.91979766 0.26677158] Correct: [68.20745463 91.6625797 ]% Cross Entropy Loss: [0.88821006 0.22378337] Correct: [69.1760667 93.02354095]% Cross Entropy Loss: [0.8804046 0.2257684] Correct: [70.15693968 92.83962727]% Cross Entropy Loss: [0.8652985 0.18099944] Correct: [70.90485532 94.96076508]% Cross Entropy Loss: [0.8585656 0.17009576] Correct: [70.26728789 95.35311427]% Cross Entropy Loss: [0.8705536 0.20302601] Correct: [70.03433055 93.83276116]% Cross Entropy Loss: [0.82587284 0.18966936] Correct: [71.49337911 94.4825895 ]% Mean abs weights tensor(0.0281, grad_fn=<MeanBackward0>)
0%| | 0/36 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 46.38% From layer 1: Accuracy: 58.92%
0%| | 0/128 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 73.25% From layer 1: Accuracy: 94.23%
0%| | 0/30 [00:00<?, ?it/s]
Cross Entropy Loss: [4.4483128 3.1161757] Correct: [15.64492398 26.79009318]% Cross Entropy Loss: [2.5584948 1.7132568] Correct: [28.65375184 47.57233938]% Cross Entropy Loss: [2.1659534 1.3345348] Correct: [34.84551251 57.41785189]% Cross Entropy Loss: [2.015003 1.0901335] Correct: [38.79352624 63.86709171]% Cross Entropy Loss: [1.8691787 1.0044407] Correct: [42.36145169 66.8710152 ]% Cross Entropy Loss: [1.7254089 0.8297699] Correct: [45.95389897 72.25355566]% Cross Entropy Loss: [1.6569808 0.74367225] Correct: [47.6949485 74.97547818]% Cross Entropy Loss: [1.5617207 0.71452576] Correct: [49.87739088 76.66748406]% Cross Entropy Loss: [1.5058035 0.60052973] Correct: [51.75331045 79.84306032]% Cross Entropy Loss: [1.4604566 0.54324216] Correct: [52.28052967 81.93967631]% Cross Entropy Loss: [1.3802576 0.5138194] Correct: [54.64688573 82.95733203]% Cross Entropy Loss: [1.3599819 0.47766754] Correct: [55.16184404 83.85237862]% Cross Entropy Loss: [1.2893304 0.4804259] Correct: [57.29524277 84.26924963]% Cross Entropy Loss: [1.2434955 0.42147464] Correct: [58.68072585 86.23099559]% Cross Entropy Loss: [1.230147 0.40258113] Correct: [59.77194703 86.70917116]% Cross Entropy Loss: [1.1664425 0.3494373] Correct: [60.34820991 88.92839627]% Cross Entropy Loss: [1.1241125 0.3357098] Correct: [61.39038744 89.02648357]% Cross Entropy Loss: [1.1038005 0.3036447] Correct: [62.22412948 90.52231486]% Cross Entropy Loss: [1.0950699 0.3065696] Correct: [62.99656694 90.47327121]% Cross Entropy Loss: [1.0616846 0.28493968] Correct: [63.80578715 90.9637077 ]% Cross Entropy Loss: [1.0119269 0.27027607] Correct: [65.06866111 91.77292791]% Cross Entropy Loss: [0.9968296 0.27108988] Correct: [65.91466405 91.8710152 ]% Cross Entropy Loss: [0.9275687 0.22652677] Correct: [67.69249632 92.91319274]% Cross Entropy Loss: [0.9371393 0.25098255] Correct: [67.52084355 91.98136341]% Cross Entropy Loss: [0.9379846 0.21601027] Correct: [68.25649828 93.41589014]% Cross Entropy Loss: [0.9290978 0.20331538] Correct: [68.0725846 94.00441393]% Cross Entropy Loss: [0.9017016 0.19907373] Correct: [68.79597842 94.20058852]% Cross Entropy Loss: [0.8746368 0.1970738] Correct: [69.56841589 93.99215302]% Cross Entropy Loss: [0.87055296 0.20123456] Correct: [69.83815596 93.80823933]% Cross Entropy Loss: [0.820621 0.16774909] Correct: [71.87346739 95.32859245]% Mean abs weights tensor(0.0282, grad_fn=<MeanBackward0>)
0%| | 0/36 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 45.32% From layer 1: Accuracy: 62.72%
0%| | 0/128 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 72.84% From layer 1: Accuracy: 96.26%
0%| | 0/30 [00:00<?, ?it/s]
Cross Entropy Loss: [4.0607557 3.0031996] Correct: [15.62040216 27.83227072]% Cross Entropy Loss: [2.5603333 1.6726999] Correct: [28.911231 48.33251594]% Cross Entropy Loss: [2.2113397 1.3044305] Correct: [35.50760177 58.1289848 ]% Cross Entropy Loss: [1.9778465 1.1184703] Correct: [39.78666013 63.90387445]% Cross Entropy Loss: [1.8704928 0.9091276] Correct: [42.9744973 69.55615498]% Cross Entropy Loss: [1.728939 0.85666287] Correct: [46.10102992 71.43207455]% Cross Entropy Loss: [1.6569219 0.71602404] Correct: [47.52329573 75.85826385]% Cross Entropy Loss: [1.550477 0.6428372] Correct: [50.02452182 78.65375184]% Cross Entropy Loss: [1.4984617 0.6055265] Correct: [51.29965669 79.47523296]% Cross Entropy Loss: [1.3847317 0.5530833] Correct: [54.34036292 81.7312408 ]% Cross Entropy Loss: [1.3731031 0.5048047] Correct: [54.46297205 82.90828838]% Cross Entropy Loss: [1.3175362 0.47308514] Correct: [57.08680726 84.83325159]% Cross Entropy Loss: [1.2559967 0.44578034] Correct: [58.00637567 85.2378617 ]% Cross Entropy Loss: [1.2617377 0.43368855] Correct: [58.0308975 85.81412457]% Cross Entropy Loss: [1.184549 0.3856602] Correct: [60.06620893 87.54291319]% Cross Entropy Loss: [1.1691493 0.36697918] Correct: [60.80186366 88.3521334 ]% Cross Entropy Loss: [1.1113313 0.33461976] Correct: [62.72682688 89.44335459]% Cross Entropy Loss: [1.0981593 0.3232142] Correct: [63.08239333 89.52918097]% Cross Entropy Loss: [1.0731611 0.3194055] Correct: [63.60961256 89.56596371]% Cross Entropy Loss: [1.0358571 0.30523935] Correct: [65.24031388 90.01961746]% Cross Entropy Loss: [1.0507315 0.26741442] Correct: [65.19127023 91.51544875]% Cross Entropy Loss: [1.0014249 0.26470965] Correct: [66.04953409 92.05492889]% Cross Entropy Loss: [0.9526882 0.22872484] Correct: [67.68023541 93.25649828]% Cross Entropy Loss: [0.94942486 0.25545955] Correct: [67.52084355 92.04266797]% Cross Entropy Loss: [0.9085201 0.23894475] Correct: [68.64884747 92.81510544]% Cross Entropy Loss: [0.93354386 0.2266892 ] Correct: [67.85188818 93.21971555]% Cross Entropy Loss: [0.86580855 0.19915287] Correct: [70.31633154 94.02893575]% Cross Entropy Loss: [0.8685792 0.24224716] Correct: [70.34085336 92.39823443]% Cross Entropy Loss: [0.833374 0.17090915] Correct: [71.16233448 94.85041687]% Cross Entropy Loss: [0.81966025 0.17884849] Correct: [71.99607651 94.81363413]% Mean abs weights tensor(0.0281, grad_fn=<MeanBackward0>)
0%| | 0/36 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 46.64% From layer 1: Accuracy: 63.03%
0%| | 0/128 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 73.06% From layer 1: Accuracy: 97.09%
0%| | 0/30 [00:00<?, ?it/s]
Cross Entropy Loss: [4.0985374 3.2944996] Correct: [16.07405591 25.698872 ]% Cross Entropy Loss: [2.455575 1.700628] Correct: [29.76949485 47.41294752]% Cross Entropy Loss: [2.2019534 1.36118 ] Correct: [35.60568906 56.74350172]% Cross Entropy Loss: [2.0125885 1.1350266] Correct: [39.33300638 63.37665522]% Cross Entropy Loss: [1.796655 0.9361459] Correct: [43.80823933 69.29867582]% Cross Entropy Loss: [1.7527888 0.84369 ] Correct: [44.94850417 71.65277097]% Cross Entropy Loss: [1.6077286 0.74455225] Correct: [48.14860226 75.49043649]% Cross Entropy Loss: [1.5420077 0.69764966] Correct: [49.85286905 76.93722413]% Cross Entropy Loss: [1.4828748 0.5889198] Correct: [51.63070132 80.08827857]% Cross Entropy Loss: [1.402006 0.5506784] Correct: [53.60470819 81.63315351]% Cross Entropy Loss: [1.351154 0.5050778] Correct: [55.17410495 83.19028936]% Cross Entropy Loss: [1.3070261 0.45732045] Correct: [56.11819519 84.99264345]% Cross Entropy Loss: [1.2810955 0.4435083] Correct: [57.24619912 85.49534085]% Cross Entropy Loss: [1.2076586 0.41869372] Correct: [59.31829328 86.2432565 ]% Cross Entropy Loss: [1.1462864 0.38581115] Correct: [61.35360471 87.28543404]% Cross Entropy Loss: [1.1318189 0.35406032] Correct: [61.53751839 88.58509073]% Cross Entropy Loss: [1.1145303 0.36534613] Correct: [62.38352133 88.03334968]% Cross Entropy Loss: [1.099712 0.3723377] Correct: [62.73908779 87.69004414]% Cross Entropy Loss: [1.0948285 0.32933536] Correct: [63.14369789 89.2594409 ]% Cross Entropy Loss: [1.003488 0.28201535] Correct: [65.25257479 90.92692496]% Cross Entropy Loss: [1.0008978 0.26445487] Correct: [65.60814125 91.42962236]% Cross Entropy Loss: [0.9835364 0.2391625] Correct: [66.34379598 92.80284453]% Cross Entropy Loss: [0.99835587 0.2664332 ] Correct: [66.71162334 91.38057872]% Cross Entropy Loss: [0.9341456 0.24310821] Correct: [68.35458558 92.63119176]% Cross Entropy Loss: [0.88600963 0.22124572] Correct: [68.80823933 93.28102011]% Cross Entropy Loss: [0.8852154 0.21745065] Correct: [69.76459049 93.23197646]% Cross Entropy Loss: [0.86673075 0.21071863] Correct: [70.36537518 93.74693477]% Cross Entropy Loss: [0.86694473 0.18618192] Correct: [70.47572339 94.5806768 ]% Cross Entropy Loss: [0.83385706 0.18601525] Correct: [71.22363904 94.28641491]% Cross Entropy Loss: [0.81904113 0.18668696] Correct: [71.62824914 94.5806768 ]% Mean abs weights tensor(0.0282, grad_fn=<MeanBackward0>)
0%| | 0/36 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 44.57% From layer 1: Accuracy: 63.34%
0%| | 0/128 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 73.85% From layer 1: Accuracy: 97.81%
0%| | 0/30 [00:00<?, ?it/s]
Cross Entropy Loss: [4.7971826 3.2862742] Correct: [13.8548308 25.31878372]% Cross Entropy Loss: [2.621397 1.7210925] Correct: [27.75870525 46.29720451]% Cross Entropy Loss: [2.2450938 1.2936238] Correct: [34.17116233 57.82246199]% Cross Entropy Loss: [2.0452032 1.131343 ] Correct: [38.13143698 62.69004414]% Cross Entropy Loss: [1.8787491 0.95713943] Correct: [41.68710152 68.34232467]% Cross Entropy Loss: [1.7834831 0.8576563] Correct: [44.39676312 71.93477195]% Cross Entropy Loss: [1.6640488 0.7480555] Correct: [46.81216282 75.17165277]% Cross Entropy Loss: [1.5685259 0.68799865] Correct: [48.98234429 77.02305051]% Cross Entropy Loss: [1.4929684 0.5893494] Correct: [51.02991663 79.85532124]% Cross Entropy Loss: [1.4721227 0.5467555] Correct: [52.42766062 82.07454635]% Cross Entropy Loss: [1.4286007 0.5760207] Correct: [53.99705738 81.17949975]% Cross Entropy Loss: [1.3564253 0.47276524] Correct: [55.98332516 84.60029426]% Cross Entropy Loss: [1.2754017 0.45800918] Correct: [57.25846003 85.21333987]% Cross Entropy Loss: [1.2299513 0.42598176] Correct: [58.70524767 86.23099559]% Cross Entropy Loss: [1.1954603 0.34833822] Correct: [59.57577244 88.96517901]% Cross Entropy Loss: [1.1853702 0.3482327] Correct: [60.22560078 88.74448259]% Cross Entropy Loss: [1.1468145 0.33548844] Correct: [61.02256008 89.29622364]% Cross Entropy Loss: [1.1210716 0.31688228] Correct: [62.33447769 89.51692006]% Cross Entropy Loss: [1.1064711 0.28780884] Correct: [62.62873958 91.03727317]% Cross Entropy Loss: [1.0739075 0.27798635] Correct: [64.21039725 91.30701324]% Cross Entropy Loss: [0.99441224 0.26528 ] Correct: [65.87788131 91.6625797 ]% Cross Entropy Loss: [0.97331583 0.25762606] Correct: [66.29475233 91.77292791]% Cross Entropy Loss: [0.96135193 0.25288576] Correct: [66.89553703 92.15301618]% Cross Entropy Loss: [0.9203157 0.21776143] Correct: [68.34232467 93.40362923]% Cross Entropy Loss: [0.9123857 0.22536948] Correct: [68.50171653 93.10936734]% Cross Entropy Loss: [0.9121338 0.20319808] Correct: [68.74693477 94.02893575]% Cross Entropy Loss: [0.8847519 0.20627722] Correct: [69.51937224 93.62432565]% Cross Entropy Loss: [0.85521233 0.2008538 ] Correct: [70.48798431 93.86954389]% Cross Entropy Loss: [0.86435163 0.1951009 ] Correct: [70.15693968 93.99215302]% Cross Entropy Loss: [0.8571837 0.16028777] Correct: [70.34085336 95.52476704]% Mean abs weights tensor(0.0282, grad_fn=<MeanBackward0>)
0%| | 0/36 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 45.54% From layer 1: Accuracy: 64.09%
0%| | 0/128 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 74.02% From layer 1: Accuracy: 95.82%
0%| | 0/30 [00:00<?, ?it/s]
Cross Entropy Loss: [4.0969534 3.2174468] Correct: [17.03040706 25.51495831]% Cross Entropy Loss: [2.429454 1.7140265] Correct: [30.88523786 47.00833742]% Cross Entropy Loss: [2.1574295 1.4132793] Correct: [36.41490927 55.16184404]% Cross Entropy Loss: [1.9218984 1.1097312] Correct: [41.57675331 63.09465424]% Cross Entropy Loss: [1.7856371 0.9692116] Correct: [44.55615498 68.28102011]% Cross Entropy Loss: [1.7504174 0.88895535] Correct: [46.12555174 71.45659637]% Cross Entropy Loss: [1.5854278 0.74244195] Correct: [49.32564983 75.1961746 ]% Cross Entropy Loss: [1.4941695 0.6737008] Correct: [52.0107896 77.57479156]% Cross Entropy Loss: [1.4568505 0.650215 ] Correct: [52.58705248 77.88131437]% Cross Entropy Loss: [1.4022188 0.55166674] Correct: [54.57332026 81.5228053 ]% Cross Entropy Loss: [1.3765494 0.5480042] Correct: [54.90436488 81.78028445]% Cross Entropy Loss: [1.2722019 0.48113486] Correct: [57.69985287 84.30603237]% Cross Entropy Loss: [1.2381637 0.4440428] Correct: [59.07307504 85.6424718 ]% Cross Entropy Loss: [1.1999272 0.40076938] Correct: [59.67385974 86.97891123]% Cross Entropy Loss: [1.1747376 0.3969127] Correct: [61.13290829 87.23639039]% Cross Entropy Loss: [1.1263044 0.352383 ] Correct: [62.37126042 88.83030897]% Cross Entropy Loss: [1.0945936 0.34725538] Correct: [62.80039235 88.78126533]% Cross Entropy Loss: [1.0927949 0.3492716] Correct: [63.10691515 88.83030897]% Cross Entropy Loss: [1.0781292 0.30218056] Correct: [63.84256989 90.71848946]% Cross Entropy Loss: [1.0408287 0.2835253] Correct: [64.94605199 91.11083865]% Cross Entropy Loss: [0.97929776 0.29249007] Correct: [66.65031878 90.64492398]% Cross Entropy Loss: [0.957546 0.27482376] Correct: [67.32466896 90.84109858]% Cross Entropy Loss: [0.9396979 0.25831527] Correct: [68.24423737 92.22658166]% Cross Entropy Loss: [0.9471148 0.24204664] Correct: [67.69249632 92.42275625]% Cross Entropy Loss: [0.93977016 0.26991615] Correct: [68.09710642 90.89014223]% Cross Entropy Loss: [0.9255674 0.24245481] Correct: [68.80823933 92.39823443]% Cross Entropy Loss: [0.880603 0.2068333] Correct: [69.99754782 93.63658656]% Cross Entropy Loss: [0.8318071 0.19322075] Correct: [71.33398725 94.1760667 ]% Cross Entropy Loss: [0.8271154 0.18292095] Correct: [72.13094654 94.69102501]% Cross Entropy Loss: [0.8139197 0.179378 ] Correct: [71.8857283 95.02206964]% Mean abs weights tensor(0.0282, grad_fn=<MeanBackward0>)
0%| | 0/36 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 43.73% From layer 1: Accuracy: 62.10%
0%| | 0/128 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 72.33% From layer 1: Accuracy: 96.65% Fast Classifier Mean Test Accuracy: tensor([45.0751, 62.0053]), Std: tensor([1.2252, 1.8781]) Fast Classifier Mean Train Accuracy: tensor([72.5552, 95.8301]), Std: tensor([1.5979, 1.4504])
0%| | 0/30 [00:00<?, ?it/s]
Cross Entropy Loss: [3.8480463 3.3947315] Correct: [17.28788622 28.07748897]% Cross Entropy Loss: [2.4616585 1.682569 ] Correct: [30.68906327 50.08582639]% Cross Entropy Loss: [2.0636897 1.2741773] Correct: [37.54291319 60.2378617 ]% Cross Entropy Loss: [1.8968732 1.0529089] Correct: [41.18440412 67.38597352]% Cross Entropy Loss: [1.7540731 0.88260776] Correct: [44.28641491 71.76311918]% Cross Entropy Loss: [1.6600736 0.8081438] Correct: [47.00833742 74.0926925 ]% Cross Entropy Loss: [1.5845402 0.7514948] Correct: [49.47278077 76.29965669]% Cross Entropy Loss: [1.4956416 0.6225226] Correct: [51.32417852 79.52427661]% Cross Entropy Loss: [1.4507385 0.6199757] Correct: [52.97940167 80.02697401]% Cross Entropy Loss: [1.384391 0.564674] Correct: [54.36488475 81.46150074]% Cross Entropy Loss: [1.3296341 0.45324314] Correct: [55.50514958 85.45855812]% Cross Entropy Loss: [1.30213 0.49778086] Correct: [57.07454635 84.50220696]% Cross Entropy Loss: [1.2610537 0.39183384] Correct: [58.04315841 87.08925944]% Cross Entropy Loss: [1.2320108 0.37599227] Correct: [59.36733693 88.00882786]% Cross Entropy Loss: [1.1600598 0.35792643] Correct: [61.41490927 88.19274154]% Cross Entropy Loss: [1.0935596 0.33121333] Correct: [62.06473762 89.63952918]% Cross Entropy Loss: [1.0809547 0.3185426] Correct: [63.97743992 90.01961746]% Cross Entropy Loss: [1.0727597 0.31754342] Correct: [63.40117705 89.93379107]% Cross Entropy Loss: [1.0573388 0.2775604] Correct: [65.20353114 91.08631682]% Cross Entropy Loss: [1.0161644 0.24532874] Correct: [65.33840118 92.38597352]% Cross Entropy Loss: [0.9696506 0.27651575] Correct: [67.22658166 91.08631682]% Cross Entropy Loss: [0.98432136 0.3462424 ] Correct: [66.62579696 89.89700834]% Cross Entropy Loss: [0.9409854 0.2473308] Correct: [68.34232467 91.98136341]% Cross Entropy Loss: [0.9183697 0.18199635] Correct: [69.11476214 94.72780775]% Cross Entropy Loss: [0.8984795 0.1811345] Correct: [69.12702305 94.20058852]% Cross Entropy Loss: [0.86999714 0.13408804] Correct: [70.3776361 95.95389897]% Cross Entropy Loss: [0.8988687 0.27052125] Correct: [69.61745954 92.14075527]% Cross Entropy Loss: [0.8641304 0.1990731] Correct: [70.53702795 93.36684649]% Cross Entropy Loss: [0.8223216 0.20762241] Correct: [71.21137813 93.19519372]% Cross Entropy Loss: [0.80306524 0.12395442] Correct: [72.64590486 96.13781265]% Mean abs weights tensor(0.0231, grad_fn=<MeanBackward0>)
0%| | 0/36 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 44.52% From layer 1: Accuracy: 63.52%
0%| | 0/128 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 72.14% From layer 1: Accuracy: 95.38%
0%| | 0/30 [00:00<?, ?it/s]
Cross Entropy Loss: [4.206088 3.5189412] Correct: [17.68023541 28.80088279]% Cross Entropy Loss: [2.4782095 1.8298502] Correct: [29.85532124 48.11181952]% Cross Entropy Loss: [2.1422782 1.4196917] Correct: [36.46395292 58.19028936]% Cross Entropy Loss: [1.960169 1.0874754] Correct: [39.70083374 64.94605199]% Cross Entropy Loss: [1.8213218 0.98169035] Correct: [43.37910741 69.33545856]% Cross Entropy Loss: [1.6789528 0.82417023] Correct: [47.00833742 73.8965179 ]% Cross Entropy Loss: [1.5771729 0.8443799] Correct: [49.11721432 74.21530162]% Cross Entropy Loss: [1.4971353 0.6665225] Correct: [50.84600294 78.5188818 ]% Cross Entropy Loss: [1.4465986 0.6179621] Correct: [52.35409514 79.78175576]% Cross Entropy Loss: [1.3983774 0.538283 ] Correct: [54.15644924 82.05002452]% Cross Entropy Loss: [1.3439674 0.5567146] Correct: [55.11280039 81.84158901]% Cross Entropy Loss: [1.3178648 0.44285196] Correct: [56.48602256 85.40951447]% Cross Entropy Loss: [1.241287 0.41059518] Correct: [58.69298676 86.45169201]% Cross Entropy Loss: [1.2061244 0.38343447] Correct: [59.78420795 87.00343306]% Cross Entropy Loss: [1.1600366 0.34978142] Correct: [60.49534085 88.92839627]% Cross Entropy Loss: [1.1510956 0.32274026] Correct: [60.45855812 89.57822462]% Cross Entropy Loss: [1.0788989 0.32024953] Correct: [62.18734674 89.63952918]% Cross Entropy Loss: [1.0813724 0.33582827] Correct: [63.41343796 89.39431094]% Cross Entropy Loss: [1.0472956 0.2853518] Correct: [64.74987739 90.60814125]% Cross Entropy Loss: [1.0207807 0.31041095] Correct: [64.72535557 89.89700834]% Cross Entropy Loss: [0.96917814 0.3011455 ] Correct: [66.49092692 90.70622854]% Cross Entropy Loss: [1.0051692 0.22701487] Correct: [66.23344777 92.31240804]% Cross Entropy Loss: [0.9415259 0.199434 ] Correct: [67.54536538 93.56302109]% Cross Entropy Loss: [0.927842 0.18669309] Correct: [68.25649828 93.99215302]% Cross Entropy Loss: [0.8912463 0.23012541] Correct: [68.95537028 92.48406081]% Cross Entropy Loss: [0.8806901 0.17567062] Correct: [69.65424228 94.32319765]% Cross Entropy Loss: [0.8703813 0.17074762] Correct: [69.89946052 94.54389407]% Cross Entropy Loss: [0.8536955 0.1719962] Correct: [70.34085336 94.47032859]% Cross Entropy Loss: [0.82452786 0.17609704] Correct: [71.59146641 94.04119667]% Cross Entropy Loss: [0.8003086 0.20986572] Correct: [72.31486023 93.73467386]% Mean abs weights tensor(0.0230, grad_fn=<MeanBackward0>)
0%| | 0/36 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 42.67% From layer 1: Accuracy: 64.97%
0%| | 0/128 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 67.48% From layer 1: Accuracy: 96.98%
0%| | 0/30 [00:00<?, ?it/s]
Cross Entropy Loss: [4.1560173 3.367989 ] Correct: [17.94997548 29.14418833]% Cross Entropy Loss: [2.590688 1.7177145] Correct: [28.96027464 49.46051986]% Cross Entropy Loss: [2.2253058 1.4117813] Correct: [35.60568906 57.6262874 ]% Cross Entropy Loss: [2.0219944 1.1347914] Correct: [39.74987739 65.21579205]% Cross Entropy Loss: [1.8264041 0.8972822] Correct: [43.66110839 71.10102992]% Cross Entropy Loss: [1.735253 0.83923924] Correct: [45.54928887 72.94016675]% Cross Entropy Loss: [1.6873991 0.8241558] Correct: [46.86120647 74.27660618]% Cross Entropy Loss: [1.5840389 0.63882774] Correct: [49.36243256 79.02157921]% Cross Entropy Loss: [1.5228869 0.6021028] Correct: [51.38548308 80.66454144]% Cross Entropy Loss: [1.43229 0.48725578] Correct: [52.67287886 83.86463953]% Cross Entropy Loss: [1.3550743 0.45841601] Correct: [55.44384502 84.66159882]% Cross Entropy Loss: [1.3344808 0.53857064] Correct: [55.9220206 82.98185385]% Cross Entropy Loss: [1.2682309 0.44500375] Correct: [57.60176557 85.26238352]% Cross Entropy Loss: [1.2394811 0.37625623] Correct: [58.34968122 87.72682688]% Cross Entropy Loss: [1.2252802 0.36405498] Correct: [58.77881314 88.4502207 ]% Cross Entropy Loss: [1.1619779 0.43156856] Correct: [61.13290829 85.7405591 ]% Cross Entropy Loss: [1.1320895 0.34387794] Correct: [61.61108386 88.59735164]% Cross Entropy Loss: [1.0682434 0.26920846] Correct: [63.53604708 91.31927415]% Cross Entropy Loss: [1.0796478 0.27721244] Correct: [63.30308975 91.35605689]% Cross Entropy Loss: [1.0644479 0.30462387] Correct: [64.4556155 90.43648847]% Cross Entropy Loss: [1.0150723 0.26614684] Correct: [65.06866111 91.78518882]% Cross Entropy Loss: [0.9981913 0.29921976] Correct: [65.5590976 90.54683668]% Cross Entropy Loss: [0.95010656 0.24726525] Correct: [67.15301618 92.00588524]% Cross Entropy Loss: [0.987556 0.21103159] Correct: [66.44188328 93.39136832]% Cross Entropy Loss: [0.92104584 0.2178937 ] Correct: [68.04806278 92.71701815]% Cross Entropy Loss: [0.8756506 0.25691646] Correct: [70.15693968 91.76066699]% Cross Entropy Loss: [0.87623113 0.16466418] Correct: [69.91172143 94.92398234]% Cross Entropy Loss: [0.87307906 0.16258323] Correct: [69.75232957 94.89946052]% Cross Entropy Loss: [0.8674003 0.16167463] Correct: [70.48798431 94.66650319]% Cross Entropy Loss: [0.81002593 0.14264874] Correct: [71.787641 95.64737616]% Mean abs weights tensor(0.0230, grad_fn=<MeanBackward0>)
0%| | 0/36 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 47.35% From layer 1: Accuracy: 65.68%
0%| | 0/128 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 73.55% From layer 1: Accuracy: 96.52%
0%| | 0/30 [00:00<?, ?it/s]
Cross Entropy Loss: [4.0714426 3.244105 ] Correct: [17.81510544 29.46297205]% Cross Entropy Loss: [2.455151 1.6656104] Correct: [31.00784698 50.23295733]% Cross Entropy Loss: [2.1909482 1.3464797] Correct: [36.03482099 58.90142227]% Cross Entropy Loss: [1.9907267 1.1786809] Correct: [39.40657185 65.31387935]% Cross Entropy Loss: [1.8365068 0.9466475] Correct: [42.99901913 69.88719961]% Cross Entropy Loss: [1.6985853 0.788624 ] Correct: [46.43207455 75.29426189]% Cross Entropy Loss: [1.5932205 0.705826 ] Correct: [48.87199608 76.98626778]% Cross Entropy Loss: [1.5121889 0.64586574] Correct: [50.51495831 79.20549289]% Cross Entropy Loss: [1.5145305 0.57844985] Correct: [50.93182933 80.88523786]% Cross Entropy Loss: [1.4117968 0.47179782] Correct: [52.93035802 84.60029426]% Cross Entropy Loss: [1.3390776 0.50047755] Correct: [55.46836685 83.938205 ]% Cross Entropy Loss: [1.2993658 0.46249327] Correct: [56.79254537 84.95586072]% Cross Entropy Loss: [1.3256488 0.44449434] Correct: [56.41245709 85.56890633]% Cross Entropy Loss: [1.2111647 0.4104222] Correct: [59.42864149 86.35360471]% Cross Entropy Loss: [1.1526072 0.3120318] Correct: [60.80186366 90.0564002 ]% Cross Entropy Loss: [1.1626122 0.31269935] Correct: [60.75282001 89.43109367]% Cross Entropy Loss: [1.124913 0.2976848] Correct: [62.00343306 90.0564002 ]% Cross Entropy Loss: [1.0740774 0.23546451] Correct: [63.52378617 92.41049534]% Cross Entropy Loss: [1.0297993 0.23527752] Correct: [65.27709662 92.60666994]% Cross Entropy Loss: [1.0638537 0.2707165] Correct: [64.39431094 90.79205493]% Cross Entropy Loss: [0.99400276 0.28485528] Correct: [65.92692496 90.43648847]% Cross Entropy Loss: [0.96417683 0.20715752] Correct: [67.01814615 92.99901913]% Cross Entropy Loss: [0.94441694 0.19730134] Correct: [67.43501717 93.66110839]% Cross Entropy Loss: [0.9213695 0.15779217] Correct: [68.91858754 95.25502697]% Cross Entropy Loss: [0.9132426 0.2743287] Correct: [68.66110839 91.20892594]% Cross Entropy Loss: [0.8812156 0.17188132] Correct: [69.20058852 94.60519863]% Cross Entropy Loss: [0.8708519 0.18731025] Correct: [70.09563512 94.0779794 ]% Cross Entropy Loss: [0.85586435 0.19063199] Correct: [70.51250613 93.6733693 ]% Cross Entropy Loss: [0.83280563 0.14246862] Correct: [71.17459539 95.48798431]% Cross Entropy Loss: [0.8053354 0.20717162] Correct: [72.2903384 93.53849926]% Mean abs weights tensor(0.0229, grad_fn=<MeanBackward0>)
0%| | 0/36 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 41.70% From layer 1: Accuracy: 61.48%
0%| | 0/128 [00:00<?, ?it/s]
--------------------------------------------------------------------------- KeyboardInterrupt Traceback (most recent call last) Input In [9], in <cell line: 1>() 14 print('Mean abs weights', out_projs[-1].out_proj.weight.abs().mean()) 15 test_accs.append(get_accuracy(SNN, out_projs, test_loader, device, cat=cat)[0]) ---> 16 train_accs.append(get_accuracy(SNN, out_projs, train_loader, device, cat=cat)[0]) 17 test_accs = torch.stack([torch.tensor(ta) for ta in test_accs]) 18 train_accs = torch.stack([torch.tensor(ta) for ta in train_accs]) File ~/ownCloud/ETH/Master/Project_2/SNN_CLAPP/utils.py:155, in get_accuracy(SNN, out_projs, dataloader, device, cat) 153 out, mem = out_proj(torch.cat(spk_step[:i+1], axis=-1), target) 154 else: --> 155 out, mem = out_proj(spk_step[i], target) 156 if step == inp.shape[1]-1: 157 logits[i] = mem File ~/miniconda3/lib/python3.9/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs) 1496 # If we don't have any hooks, we want to skip the rest of the logic in 1497 # this function, and just call forward. 1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks 1499 or _global_backward_pre_hooks or _global_backward_hooks 1500 or _global_forward_hooks or _global_forward_pre_hooks): -> 1501 return forward_call(*args, **kwargs) 1502 # Do not call functions when jit is used 1503 full_backward_hooks, non_full_backward_hooks = [], [] File ~/ownCloud/ETH/Master/Project_2/SNN_CLAPP/model.py:298, in simple_out.forward(self, inp, target) 296 def forward(self, inp, target): 297 cur = self.out_proj(inp) --> 298 spk, self.mem = self.lif(cur, self.mem) 299 if self.training: 300 self._update_trace(inp) File ~/miniconda3/lib/python3.9/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs) 1496 # If we don't have any hooks, we want to skip the rest of the logic in 1497 # this function, and just call forward. 1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks 1499 or _global_backward_pre_hooks or _global_backward_hooks 1500 or _global_forward_hooks or _global_forward_pre_hooks): -> 1501 return forward_call(*args, **kwargs) 1502 # Do not call functions when jit is used 1503 full_backward_hooks, non_full_backward_hooks = [], [] File ~/miniconda3/lib/python3.9/site-packages/snntorch/_neurons/leaky.py:180, in Leaky.forward(self, input_, mem) 173 # TO-DO: alternatively, we could do torch.exp(-1 / 174 # self.beta.clamp_min(0)), 175 # giving actual time constants instead of values in [0, 1] as 176 # initial beta 177 # beta = self.beta.clamp(0, 1) 179 if not self.init_hidden: --> 180 self.reset = self.mem_reset(mem) 181 mem = self._build_state_function(input_, mem) 183 if self.state_quant: File ~/miniconda3/lib/python3.9/site-packages/snntorch/_neurons/neurons.py:106, in SpikingNeuron.mem_reset(self, mem) 103 def mem_reset(self, mem): 104 """Generates detached reset signal if mem > threshold. 105 Returns reset.""" --> 106 mem_shift = mem - self.threshold 107 reset = self.spike_grad(mem_shift).clone().detach() 109 return reset KeyboardInterrupt:
# run all checkpoints
if not args.augment:
last_ckp = 20
test_accs_ckpt = []
train_accs_ckpt = []
epochs = []
while True:
print(f'Checkpoint {last_ckp}')
SNN_ckp = EchoSpike(args.n_inputs, args.n_hidden, beta=args.beta, device=device, recurrency_type=args.recurrency_type, online=args.online).to(device)
try:
print(model_name[:-3] + f'_epoch{last_ckp}.pt')
state_dict = torch.load(folder+model_name[:-3] + f'_epoch{last_ckp}.pt', map_location=device)
# state_dict = {key.replace('clapp', 'layers'):value for key, value in state_dict.items()}
# torch.save(state_dict, folder+model_name + f'_epoch{last_ckp}.pt')
SNN_ckp.load_state_dict(state_dict)
except:
if last_ckp > 1500:
break
else:
last_ckp += 20
continue
epochs.append(last_ckp)
last_ckp += 20
snn_samples, targets = get_samples(SNN_ckp, train_loader, args.n_hidden, device)
cat = True
with torch.no_grad():
out_projs, acc, losses_out = train_out_proj_fast(SNN_ckp, args, 60, 60, snn_samples, train_loader.y, cat=cat, lr=1e-4, weight_decay=1)
test_accs_ckpt.append(torch.tensor(get_accuracy(SNN_ckp, out_projs, test_loader, device, cat=cat)[0]))
train_accs_ckpt.append(torch.tensor(get_accuracy(SNN_ckp, out_projs, train_loader, device, cat=cat)[0]))
test_accs_ckpt = torch.stack(test_accs_ckpt)
train_accs_ckpt = torch.stack(train_accs_ckpt)
# save the results
torch.save(torch.stack([torch.tensor(tac) for tac in test_accs_ckpt]), folder+model_name[:-3]+'_test_accs_ckpt.pt')
torch.save(torch.stack([torch.tensor(tac) for tac in train_accs_ckpt]), folder+model_name[:-3]+'_train_accs_ckpt.pt')
# plot train and test accuracy over time
plt.figure()
for i in range(test_accs_ckpt.shape[-1]):
if i == 0:
plt.plot(epochs, 100*test_accs_ckpt[:,i], color=color_list[i], label=f'Directly from inputs')
plt.plot(epochs, 100*train_accs_ckpt[:,i], color=color_list[i], linestyle='--')
else:
plt.plot(epochs, 100*test_accs_ckpt[:,i], color=color_list[i], label=f'Layer {i}')
plt.plot(epochs, 100*train_accs_ckpt[:,i], color=color_list[i], linestyle='--')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()
<matplotlib.legend.Legend at 0x7f4d60de9850>
if not args.augment:
# if snn_samples already exist, don't recompute
try:
snn_samples
except:
snn_samples, targets = get_samples(SNN, train_loader, args.n_hidden, device)
for cat in [False, True]:
out_projs_closed = train_out_proj_closed_form(args, snn_samples, targets, cat=cat)
test_acc_closed, _ = get_accuracy(SNN, out_projs_closed, test_loader, device, cat=cat)
train_acc_closed, _ = get_accuracy(SNN, out_projs_closed, train_loader, device, cat=cat)
# grouped Bar plot the Accuracies of the different layers both during training and testing
sns.set_theme(style="whitegrid")
labels = ['From Inputs Directly', *[f'Until Layer {i+1}' for i in range(len(SNN.layers))]]
if not cat:
labels = ['From Inputs Directly', *[f'From Layer {i+1}' for i in range(len(SNN.layers))]]
x = np.arange(len(labels)) # the label locations
width = 0.35 # the width of the bars
fig, ax = plt.subplots()
rects1 = ax.bar(x - width/2, 100*torch.tensor(test_acc_closed), width, label='Test Accuracy', color=color_list[0])
rects2 = ax.bar(x + width/2, 100*torch.tensor(train_acc_closed), width, label='Train Accuracy', color=color_list[1])
# remove horizontal lines and spines
ax.spines['right'].set_visible(False)
ax.spines['left'].set_visible(False)
ax.xaxis.grid(False)
plt.xticks(np.arange(len(out_projs_closed)), labels, rotation=45)
plt.legend()
plt.ylabel('Accuracy [%]')
plt.ylim([25, 100])
if cat:
accs['train_closed_cat'] = train_acc_closed
accs['test_closed_cat'] = test_acc_closed
else:
accs['train_closed'] = train_acc_closed
accs['test_closed'] = test_acc_closed
with open('final_results/' + model_name[:-3] + '_accuracies.pkl', 'wb') as f:
pickle.dump(accs, f)
(20, 700) 0.010724677 -0.010629931 (20, 450) 0.15072547 -0.08645948 (20, 450) 0.35159343 -0.3516128 (20, 450) 20.087902 -23.734768
0%| | 0/36 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 28.89% From layer 1: Accuracy: 61.48% From layer 2: Accuracy: 70.89% From layer 3: Accuracy: 70.19%
0%| | 0/128 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 66.96% From layer 1: Accuracy: 84.22% From layer 2: Accuracy: 87.21% From layer 3: Accuracy: 80.90% (20, 700) 0.010724677 -0.010629931 (20, 1150) 0.15413637 -0.10144029 (20, 1600) 0.2811037 -0.36118773 (20, 2050) 0.64799124 -0.65297925
0%| | 0/36 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 28.89% From layer 1: Accuracy: 61.93% From layer 2: Accuracy: 73.59% From layer 3: Accuracy: 75.57%
0%| | 0/128 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 66.96% From layer 1: Accuracy: 94.26% From layer 2: Accuracy: 97.00% From layer 3: Accuracy: 97.89%
if args.augment:
# plot some training characteristics
print(f'Accuracy of last quarter: {100*acc[-len(acc)//4:].mean(axis=0)}%')
plt.figure()
for i in range(len(acc[0])):
plt.plot(np.asarray(acc)[:,i]*100, color=color_list[i])
plt.ylabel('Accuracy [%]')
plt.xlabel('Training Step [x500]')
labels = ['From Inputs directly', *[f'From Layer {i+1}' for i in range(len(SNN.layers))]]
plt.legend(labels)
plt.ylim([65, 95])
plt.figure()
print(losses_out.shape)
for i in range(losses_out.shape[1]):
plt.plot(np.arange(len(losses_out))/len(train_loader), savgol_filter(losses_out[:,i], 99, 1), label=labels[i], color=color_list[i])
plt.ylabel('Cross Entropy Loss')
plt.xlabel('Training Step')
plt.ylim([0.15, 1.0])
plt.legend();
test_acc, pred_matrix = get_accuracy(SNN, out_projs, test_loader, device, cat=cat)
plt.figure()
plt.plot(100*np.asarray(test_acc))
plt.ylabel('Accuracy [%]')
plt.xlabel('Layer')
plt.figure()
plt.imshow(pred_matrix, origin='lower')
plt.title('Prediction Matrix for the final layer')
plt.xlabel('Prediction')
plt.ylabel('Target')
plt.xticks([i for i in range(args.n_outputs)])
plt.yticks([i for i in range(args.n_outputs)])
plt.colorbar();
0%| | 0/36 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 48.63% From layer 1: Accuracy: 72.00% From layer 2: Accuracy: 80.70% From layer 3: Accuracy: 82.86% From layer 4: Accuracy: 84.32%
from utils import get_accuracy
if args.augment:
test_acc, _ = get_accuracy(SNN, out_projs, test_loader, device, cat=cat)
train_acc, _ = get_accuracy(SNN, out_projs, train_loader, device, cat=cat)
else:
test_acc = torch.mean(test_accs, dim=0)
print(test_acc)
train_acc = torch.mean(train_accs, dim=0)
# grouped Bar plot the Accuracies of the different layers both during training and testing
sns.set_theme(style="whitegrid")
labels = ['From Inputs Directly', *[f'Until Layer {i+1}' for i in range(len(SNN.layers))]]
if not cat:
labels = ['From Inputs Directly', *[f'From Layer {i+1}' for i in range(len(SNN.layers))]]
x = np.arange(len(labels)) # the label locations
width = 0.35 # the width of the bars
fig, ax = plt.subplots()
rects1 = ax.bar(x - width/2, 100*torch.tensor(test_acc), width, label='Test Accuracy', color=color_list[0])
rects2 = ax.bar(x + width/2, 100*torch.tensor(train_acc), width, label='Train Accuracy', color=color_list[1])
if not args.augment:
ax.errorbar(x - width/2, 100*test_acc, yerr=100*torch.std(test_accs, dim=0), fmt='none', capsize=6, color=color_list[3])
ax.errorbar(x + width/2, 100*train_acc, yerr=100*torch.std(train_accs, dim=0), fmt='none', capsize=6, color=color_list[3])
# remove horizontal lines and spines
ax.spines['right'].set_visible(False)
ax.spines['left'].set_visible(False)
ax.xaxis.grid(False)
plt.xticks(np.arange(len(out_projs)), labels, rotation=45)
plt.legend()
plt.ylabel('Accuracy [%]')
plt.ylim([25, 100])
#plt.title('SHD Accuracy');
0%| | 0/36 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 48.63% From layer 1: Accuracy: 72.00% From layer 2: Accuracy: 80.70% From layer 3: Accuracy: 82.86% From layer 4: Accuracy: 84.32%
0%| | 0/128 [00:00<?, ?it/s]
Directly from inputs: Accuracy: 45.95% From layer 1: Accuracy: 78.62% From layer 2: Accuracy: 86.68% From layer 3: Accuracy: 88.97% From layer 4: Accuracy: 90.45%
(25.0, 100.0)
# Randomly select k sample of each class and save the spiking activity
n_outputs = 20
n_repeats = 1
k = 20
fewshot_accuracies = torch.zeros((n_repeats, len(SNN.layers)))
for n in range(n_repeats):
SNN.reset(0)
one_shot_samples = torch.zeros(n_outputs, n_time_bins, n_inputs)
one_shot_spks = torch.zeros(n_outputs, len(SNN.layers), n_hidden[0])
for i in trange(n_outputs):
for j in range(k):
img, _ = train_loader.next_item(i, contrastive=False)
one_shot_samples[i] = img.squeeze()
for t in range(n_time_bins):
logits, mem_his, clapp_loss = SNN(img[t].float(), 0)
one_shot_spks[i] += torch.stack(logits).squeeze()
def metric(spk, one_shot):
dists = torch.zeros(spk.shape[0], args.n_outputs)
for i in range(args.n_outputs):
one_shot_i = one_shot[i] / one_shot[i].sum()
dists[:, i] = torch.einsum('bi, i->b' , spk, one_shot_i)
return dists
def get_predictions(spks):
preds = torch.zeros(len(spks), spks[0].shape[0])
# for each layer get the prediction
for i in range(len(spks)):
dists = metric(spks[i], one_shot_spks[:,i])
preds[i] = dists.argmax(axis=-1)
return preds
dataset = test_loader
batch = int(len(dataset)/100)
correct_oneshot = torch.zeros(len(SNN.layers))
SNN.eval()
pred_matrix_oneshot = torch.zeros(n_outputs, n_outputs)
for idx in trange(0, len(dataset), batch):
SNN.reset(0)
inp, target = dataset.x[idx:idx+batch], dataset.y[idx:idx+batch]
logits = torch.zeros(len(SNN.layers), inp.shape[0], n_hidden[0])
for step in range(inp.shape[1]):
data_step = inp[:,step].float().to(device)
spk_step, _, _ = SNN(data_step, 0)
logits += torch.stack(spk_step)
preds = get_predictions(logits)
for i in range(preds.shape[0]):
correct_oneshot[i] += int((preds[i] == target).sum())
# for the last layer create the prediction matrix
for j in range(preds.shape[1]):
pred_matrix_oneshot[int(target[j]), int(preds[-1, j])] += 1
correct_oneshot /= len(dataset)
for i in range(len(SNN.layers)):
print(f'From layer {i+1}:')
print(f'Accuracy: {100*correct_oneshot[i]:.2f}%')
fewshot_accuracies[n, i] = correct_oneshot[i]
plt.imshow(pred_matrix_oneshot, origin='lower')
plt.title('Prediction Matrix for the final layer')
plt.xlabel('Prediction')
plt.ylabel('Target')
plt.xticks([i for i in range(n_outputs)])
plt.yticks([i for i in range(n_outputs)])
plt.colorbar();
plt.figure()
# Boxplot of the accuracies
plt.figure()
sns.set_style("whitegrid")
g = sns.boxplot(data=fewshot_accuracies*100)
# remove left spines
sns.despine(left=True)
plt.xticks(np.arange(len(SNN.layers)), [f'Layer {i+1}' for i in range(len(SNN.layers))])
plt.ylabel('Few-Shot Test Accuracy [%]')
plt.ylim([0, 100])
print(f'Average Accuracy: {100*fewshot_accuracies.mean(axis=0)}%')
print(f'Maximum Accuracy: {fewshot_accuracies.max(axis=0)}%')